{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:39:45.395362Z",
     "start_time": "2017-06-16T20:39:37.929016Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",35)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:39:47.106313Z",
     "start_time": "2017-06-16T20:39:45.396887Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train__2labels.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "    kdd_test__2labels = pd.read_pickle(\"dataset/kdd_test__2labels.pkl\")\n",
    "    \n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:39:47.113072Z",
     "start_time": "2017-06-16T20:39:47.107844Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25192, 124)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:39:47.121001Z",
     "start_time": "2017-06-16T20:39:47.114489Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22544, 124)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:39:52.305761Z",
     "start_time": "2017-06-16T20:39:47.122408Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97509982675167528"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Normal','is_Attack']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "    \n",
    "    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "    x_test_ = ss.transform(x_test__input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test.values\n",
    "    y_test_ = y_test_.values\n",
    "\n",
    "    \n",
    "preprocess.x_train.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:40:12.388070Z",
     "start_time": "2017-06-16T20:39:52.307202Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.contrib.legacy_seq2seq.python.ops.seq2seq import basic_rnn_seq2seq\n",
    "from tensorflow.contrib.rnn import RNNCell, GRUCell, MultiRNNCell\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:40:12.648690Z",
     "start_time": "2017-06-16T20:40:12.389907Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 122\n",
    "    classes = 2\n",
    "    hidden_encoder_dim = 122\n",
    "    hidden_layers = 1\n",
    "    latent_dim = 10\n",
    "\n",
    "    hidden_decoder_dim = 122\n",
    "    lam = 0.01\n",
    "    \n",
    "    def __init__(self, classes, hidden_layers, num_of_features):\n",
    "        self.classes = classes\n",
    "        self.hidden_layers = hidden_layers\n",
    "        self.latent_dim = num_of_features\n",
    "            \n",
    "    def build_layers(self):\n",
    "        tf.reset_default_graph()\n",
    "        #learning_rate = tf.Variable(initial_value=0.001)\n",
    "\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_encoder_dim = self.hidden_encoder_dim\n",
    "        hidden_layers = self.hidden_layers\n",
    "        latent_dim = self.latent_dim\n",
    "        hidden_decoder_dim = self.hidden_decoder_dim\n",
    "        lam = self.lam\n",
    "        \n",
    "        with tf.variable_scope(\"Input\"):\n",
    "            self.x_input = tf.placeholder(\"float\", shape=[None, 1, input_dim])\n",
    "            self.y_input_ = tf.placeholder(\"float\", shape=[None, 1, classes])\n",
    "            self.keep_prob = tf.placeholder(\"float\")\n",
    "            self.lr = tf.placeholder(\"float\")\n",
    "            self.x_list = tf.unstack(self.x_input, axis= 1)\n",
    "            self.y_list_ = tf.unstack(self.y_input_, axis = 1)\n",
    "            self.y_ = self.y_list_[0]\n",
    "            \n",
    "            #GO = tf.fill((tf.shape(self.x)[0], 1), 0.5)\n",
    "            \n",
    "            #y_with_GO = tf.stack([self.y_, GO])\n",
    "            \n",
    "        with tf.variable_scope(\"lstm\"):\n",
    "            multi_cell = MultiRNNCell([GRUCell(input_dim) for i in range(hidden_layers)] )\n",
    "            \n",
    "            self.y, states = basic_rnn_seq2seq(self.x_list, self.y_list_, multi_cell)\n",
    "            #self.y = tf.slice(self.y, [0, 0], [-1,2])\n",
    "            \n",
    "            #self.out = tf.squeeze(self.y)\n",
    "            \n",
    "            #self.y = tf.layers.dense(self.y[0], classes, activation = None)\n",
    "            \n",
    "            self.y = tf.slice(self.y[0], [0, 0], [-1,2])\n",
    "            \n",
    "        with tf.variable_scope(\"Loss\"):\n",
    "            \n",
    "            self.regularized_loss = tf.losses.mean_squared_error(self.y_, self.y)\n",
    "            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))\n",
    "            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        with tf.variable_scope(\"Optimizer\"):\n",
    "            learning_rate=self.lr\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))\n",
    "            gradients = [\n",
    "                None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "                for gradient in gradients]\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "            #self.train_op = optimizer.minimize(self.regularized_loss)\n",
    "            \n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        self.pred = tf.argmax(self.y, axis = 1)\n",
    "        self.actual = tf.argmax(self.y_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-01T00:59:00.684124Z",
     "start_time": "2017-06-01T00:58:59.843181Z"
    }
   },
   "source": [
    "batch_iterations = 200\n",
    "\n",
    "x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                          preprocess.y_train, \n",
    "                                                                          test_size=0.1)\n",
    "batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                           batch_iterations)\n",
    "                                                                          \n",
    "for i in batch_indices:\n",
    "    print(x_train[i,np.newaxis,:])\n",
    "    print(y_train[i,np.newaxis,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:40:12.948541Z",
     "start_time": "2017-06-16T20:40:12.650361Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "class Train:    \n",
    "    \n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'test_score_20', 'time_taken'])\n",
    "\n",
    "    predictions = {}\n",
    "\n",
    "    results = []\n",
    "    best_acc = 0\n",
    "    best_acc_global = 0\n",
    "\n",
    "    def train(epochs, net, h,f, lrs):\n",
    "        batch_iterations = 200\n",
    "        train_loss = None\n",
    "        Train.best_acc = 0\n",
    "        os.makedirs(\"dataset/tf_GRU_nsl_kdd-orig-/hidden layers_{}_features count_{}\".format(h,f),\n",
    "                    exist_ok = True)\n",
    "        with tf.Session() as sess:\n",
    "            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            start_time = time.perf_counter()\n",
    "            \n",
    "            accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1})\n",
    "            \n",
    "            print(\"Initial Accuracy, before training: {}\".format(accuracy))\n",
    "            \n",
    "            for c, lr in enumerate(lrs):\n",
    "                for epoch in range(1, (epochs+1)):\n",
    "                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                              preprocess.y_train, \n",
    "                                                                              test_size=0.1)\n",
    "                    batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                               batch_iterations)\n",
    "\n",
    "                    for i in batch_indices:\n",
    "\n",
    "                        _, train_loss = sess.run([net.train_op, net.regularized_loss], #net.summary_op\n",
    "                                                              feed_dict={net.x_input: x_train[i,np.newaxis,:], \n",
    "                                                                         net.y_input_: y_train[i,np.newaxis,:], \n",
    "                                                                         net.keep_prob:1, net.lr:lr})\n",
    "                        #summary_writer_train.add_summary(summary_str, epoch)\n",
    "                        if(train_loss > 1e9):\n",
    "                            print(\"Step {} | Training Loss: {:.6f}\".format(epoch, train_loss))\n",
    "\n",
    "\n",
    "                    valid_accuracy,valid_loss = sess.run([net.tf_accuracy, net.regularized_loss], #net.summary_op \n",
    "                                                          feed_dict={net.x_input: x_valid[:,np.newaxis,:], \n",
    "                                                                     net.y_input_: y_valid[:,np.newaxis,:], \n",
    "                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "\n",
    "\n",
    "                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "                    \n",
    "                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x_input: preprocess.x_test_[:,np.newaxis,:], \n",
    "                                                                             net.y_input_: preprocess.y_test_[:,np.newaxis,:], \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "\n",
    "                    print(\"Step {} | Training Loss: {:.6f} | Train Accuracy: {:.6f} | Test Accuracy: {:.6f}, {:.6f}\".format(epoch, train_loss, valid_accuracy, accuracy, accuracy_))\n",
    "\n",
    "                    if accuracy > Train.best_acc_global:\n",
    "                                Train.best_acc_global = accuracy\n",
    "                                Train.pred_value = pred_value\n",
    "                                Train.actual_value = actual_value\n",
    "                                Train.pred_value_ = pred_value_\n",
    "                                Train.actual_value_ = actual_value_\n",
    "                                Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "\n",
    "                    if accuracy > Train.best_acc:\n",
    "\n",
    "                        #net.saver.save(sess, \"dataset/tf_vae_only_nsl_kdd_hidden layers_{}_features count_{}\".format(epochs,h,f))\n",
    "                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))\n",
    "                        #curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1]})\n",
    "                        #Train.predictions.update({\"{}_{}_{}\".format(epochs,f,h):curr_pred})\n",
    "\n",
    "                        Train.best_acc = accuracy\n",
    "                        if not (np.isnan(train_loss)):\n",
    "                            net.saver.save(sess, \n",
    "                                       \"dataset/tf_GRU_nsl_kdd-orig-/hidden layers_{}_features count_{}/model\"\n",
    "                                       .format(h,f), \n",
    "                                       global_step = epoch, \n",
    "                                       write_meta_graph=False)\n",
    "\n",
    "                        curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1], \"Prediction\":pred_value})\n",
    "                        Train.predictions.update({\"{}_{}_{}\".format((epochs+1)* (c+1),f,h):\n",
    "                                                  (curr_pred, \n",
    "                                                   Train.result((epochs+1)*(c+1), f, h,valid_accuracy, accuracy, accuracy_, time.perf_counter() - start_time))})\n",
    "\n",
    "\n",
    "\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.656958Z",
     "start_time": "2017-06-16T20:40:12.950239Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Initial Accuracy, before training: 0.3846699893474579\n",
      "Step 1 | Training Loss: 0.004211 | Train Accuracy: 0.990873 | Test Accuracy: 0.789390, 0.599325\n",
      "Step 2 | Training Loss: 0.000166 | Train Accuracy: 0.999603 | Test Accuracy: 0.900594, 0.810886\n",
      "Step 3 | Training Loss: 0.000126 | Train Accuracy: 0.999603 | Test Accuracy: 0.929915, 0.866667\n",
      "Step 4 | Training Loss: 0.000321 | Train Accuracy: 0.999603 | Test Accuracy: 0.928983, 0.864895\n",
      "Step 5 | Training Loss: 0.000138 | Train Accuracy: 0.999603 | Test Accuracy: 0.939851, 0.885570\n",
      "Step 1 | Training Loss: 0.000021 | Train Accuracy: 1.000000 | Test Accuracy: 0.939851, 0.885570\n",
      "Step 2 | Training Loss: 0.000021 | Train Accuracy: 1.000000 | Test Accuracy: 0.939141, 0.884219\n",
      "Step 3 | Training Loss: 0.000011 | Train Accuracy: 1.000000 | Test Accuracy: 0.937190, 0.880506\n",
      "Step 4 | Training Loss: 0.000025 | Train Accuracy: 0.999603 | Test Accuracy: 0.937544, 0.881181\n",
      "Step 5 | Training Loss: 0.000012 | Train Accuracy: 1.000000 | Test Accuracy: 0.934972, 0.876287\n",
      "Step 1 | Training Loss: 0.000009 | Train Accuracy: 1.000000 | Test Accuracy: 0.936036, 0.878312\n",
      "Step 2 | Training Loss: 0.000013 | Train Accuracy: 1.000000 | Test Accuracy: 0.933863, 0.874177\n",
      "Step 3 | Training Loss: 0.000016 | Train Accuracy: 1.000000 | Test Accuracy: 0.932665, 0.871899\n",
      "Step 4 | Training Loss: 0.000012 | Train Accuracy: 1.000000 | Test Accuracy: 0.931334, 0.869367\n",
      "Step 5 | Training Loss: 0.000011 | Train Accuracy: 1.000000 | Test Accuracy: 0.931867, 0.870380\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n",
      "Initial Accuracy, before training: 0.5723474025726318\n",
      "Step 1 | Training Loss: 0.000193 | Train Accuracy: 0.994444 | Test Accuracy: 0.958659, 0.921603\n",
      "Step 2 | Training Loss: 0.000002 | Train Accuracy: 0.999603 | Test Accuracy: 0.965401, 0.934177\n",
      "Step 3 | Training Loss: 0.000002 | Train Accuracy: 0.999603 | Test Accuracy: 0.986249, 0.973840\n",
      "Step 4 | Training Loss: 0.000006 | Train Accuracy: 1.000000 | Test Accuracy: 0.998181, 0.996540\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.998314, 0.996793\n",
      "Step 1 | Training Loss: 0.000003 | Train Accuracy: 1.000000 | Test Accuracy: 0.998270, 0.996709\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998226, 0.996624\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998093, 0.996371\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998226, 0.996624\n",
      "Step 5 | Training Loss: 0.000001 | Train Accuracy: 1.000000 | Test Accuracy: 0.998048, 0.996287\n",
      "Step 1 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998093, 0.996371\n",
      "Step 2 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998137, 0.996456\n",
      "Step 3 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998093, 0.996371\n",
      "Step 4 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998137, 0.996456\n",
      "Step 5 | Training Loss: 0.000000 | Train Accuracy: 1.000000 | Test Accuracy: 0.998093, 0.996371\n",
      "Current Layer Attributes - epochs:5 hidden layers:5 features count:1\n",
      "Initial Accuracy, before training: 0.5530518293380737\n",
      "Step 1 | Training Loss: 0.026148 | Train Accuracy: 0.971825 | Test Accuracy: 0.776038, 0.575105\n",
      "Step 2 | Training Loss: 0.017112 | Train Accuracy: 0.983730 | Test Accuracy: 0.793825, 0.609030\n",
      "Step 3 | Training Loss: 0.010910 | Train Accuracy: 0.979365 | Test Accuracy: 0.764638, 0.553333\n",
      "Step 4 | Training Loss: 0.000122 | Train Accuracy: 0.986508 | Test Accuracy: 0.767255, 0.558397\n",
      "Step 5 | Training Loss: 0.013918 | Train Accuracy: 0.981746 | Test Accuracy: 0.769828, 0.564894\n",
      "Step 1 | Training Loss: 0.022002 | Train Accuracy: 0.980952 | Test Accuracy: 0.769163, 0.560928\n",
      "Step 2 | Training Loss: 0.016983 | Train Accuracy: 0.985714 | Test Accuracy: 0.770715, 0.565316\n",
      "Step 3 | Training Loss: 0.005196 | Train Accuracy: 0.988889 | Test Accuracy: 0.784643, 0.591561\n",
      "Step 4 | Training Loss: 0.010268 | Train Accuracy: 0.990476 | Test Accuracy: 0.791341, 0.604473\n",
      "Step 5 | Training Loss: 0.026180 | Train Accuracy: 0.982937 | Test Accuracy: 0.776437, 0.575021\n",
      "Step 1 | Training Loss: 0.000853 | Train Accuracy: 0.981349 | Test Accuracy: 0.795289, 0.612068\n",
      "Step 2 | Training Loss: 0.008635 | Train Accuracy: 0.990873 | Test Accuracy: 0.773909, 0.570042\n",
      "Step 3 | Training Loss: 0.005237 | Train Accuracy: 0.990079 | Test Accuracy: 0.790499, 0.603038\n",
      "Step 4 | Training Loss: 0.008661 | Train Accuracy: 0.989683 | Test Accuracy: 0.789878, 0.600422\n",
      "Step 5 | Training Loss: 0.000305 | Train Accuracy: 0.992857 | Test Accuracy: 0.790587, 0.601857\n"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "    features_arr = [1] #[4, 8, 16, 32]\n",
    "    hidden_layers_arr = [1, 3, 5]\n",
    "\n",
    "    epochs = [5]\n",
    "    lrs = [1e-2, 1e-2/2, 1e-2/2]\n",
    "    \n",
    "    for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "        print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "        n = network(2,h,f)\n",
    "        n.build_layers()\n",
    "        Train.train(e, n, h,f, lrs)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.663325Z",
     "start_time": "2017-06-16T20:42:19.658706Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dict1 = {}\n",
    "dict2 = []\n",
    "for k, (v1, v2) in Train.predictions.items():\n",
    "    dict1.update({k: v1})\n",
    "    dict2.append(v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.705235Z",
     "start_time": "2017-06-16T20:42:19.664693Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Train.predictions = dict1\n",
    "Train.results = dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.713208Z",
     "start_time": "2017-06-16T20:42:19.707137Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df_results = pd.DataFrame(Train.results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.732327Z",
     "start_time": "2017-06-16T20:42:19.714867Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.998314</td>\n",
       "      <td>0.996793</td>\n",
       "      <td>13.644962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score  \\\n",
       "1      6               1              3          1.0    0.998314   \n",
       "\n",
       "   test_score_20  time_taken  \n",
       "1       0.996793   13.644962  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "g = df_results.groupby(by=['no_of_features'])\n",
    "idx = g['test_score'].transform(max) == df_results['test_score']\n",
    "df_results[idx].sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.746607Z",
     "start_time": "2017-06-16T20:42:19.734075Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.998314</td>\n",
       "      <td>0.996793</td>\n",
       "      <td>13.644962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.999603</td>\n",
       "      <td>0.939851</td>\n",
       "      <td>0.885570</td>\n",
       "      <td>4.970460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.981349</td>\n",
       "      <td>0.795289</td>\n",
       "      <td>0.612068</td>\n",
       "      <td>46.292815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0.983730</td>\n",
       "      <td>0.793825</td>\n",
       "      <td>0.609030</td>\n",
       "      <td>9.089080</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   epoch  no_of_features  hidden_layers  train_score  test_score  \\\n",
       "1      6               1              3     1.000000    0.998314   \n",
       "0      6               1              1     0.999603    0.939851   \n",
       "3     18               1              5     0.981349    0.795289   \n",
       "2      6               1              5     0.983730    0.793825   \n",
       "\n",
       "   test_score_20  time_taken  \n",
       "1       0.996793   13.644962  \n",
       "0       0.885570    4.970460  \n",
       "3       0.612068   46.292815  \n",
       "2       0.609030    9.089080  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_results.sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.790813Z",
     "start_time": "2017-06-16T20:42:19.748300Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_GRU_nsl_kdd_predictions-.pkl\")\n",
    "df_results.to_pickle(\"dataset/tf_GRU_nsl_kdd_scores-.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:19.854048Z",
     "start_time": "2017-06-16T20:42:19.792367Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        print('Confusion matrix, without normalization')\n",
    "\n",
    "    print(cm)\n",
    "\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, cm[i, j].round(4),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, preprocess.output_columns_2labels, normalize = False,\n",
    "                         title = Train.best_parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:20.160056Z",
     "start_time": "2017-06-16T20:42:19.855566Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[ 9673    38]\n",
      " [    0 12833]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbcAAAGeCAYAAAAXNE8VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmclWXdx/HPd2ZkEwFxIQQUF9KU3FD0sfLBNMXdXCnX\nNHErLTOXzNTnCTW1XCo1fDTQTEWt1HK3zBUQd8QNV0AUEAWVfeb3/HFfg4eJWRjOzJm5z/ft67zm\nnOvernMY53d+v/u670sRgZmZWZ5UlLoDZmZmxebgZmZmuePgZmZmuePgZmZmuePgZmZmuePgZmZm\nuePgZmZmuePgZmZmuePgZmZmuVNV6g6YmVlxVXZbL2LJ/KLtL+bPvD8ihhZth63Awc3MLGdiyXw6\nbnxw0fa34Pnfr1m0nbUSBzczs9wRqLzPOpX3uzczs1xy5mZmljcCpFL3oqQc3MzM8shlSTMzs3xx\n5mZmlkcuS5qZWb54tGR5v3szM8slZ25mZnnksqSZmeWKcFmy1B0wMzMrNmduZma5I5clS90BMzNr\nAS5LmpmZ5YszNzOzPHJZ0szM8sUXcZf3uzczs1xy5mZmljee8sbBzcwsl1yWNDMzyxcHNzOz3EkD\nSor1aOxo0vWSZkiaWNB2iaRXJb0o6a+SehQsO0vSZEmvSdqtoH2QpJfSsiulrLYqqaOkW1P7OEn9\nG+uTg5uZWR5VqHiPxo0ChtZpexAYGBGbA68DZwFI2hQYBmyWtrlKUmXa5mrgWGBAetTu8xjg44jY\nCLgM+FWjb78pvTYzM6tPRDwKzK7T9kBELEkvxwJ90/N9gVsiYmFEvA1MBgZL6g10i4ixERHADcB+\nBduMTs9vB3auzerq4wElZmZ5U/xZAdaUNKHg9ciIGLkC2x8N3Jqe9yELdrWmprbF6Xnd9tptpgBE\nxBJJc4A1gFn1HdDBzcwsj4p7KcCsiNimed3Q2cAS4KZidqgxLkuamVmLkHQUsBdwaCo1AkwD+hWs\n1je1TeOL0mVh+zLbSKoCugMfNXRsBzczs9xp3dGSy+2BNBQ4HdgnIuYVLLoLGJZGQK5PNnBkfERM\nB+ZK2j6dTzsCuLNgmyPT8wOBfxYEy+VyWdLMzFaKpJuBIWTn5qYC55KNjuwIPJjGfoyNiOMj4mVJ\nY4BJZOXKkyKiOu3qRLKRl52Be9MD4DrgRkmTyQauDGu0T40EPzMza2cquvWNjtv9sGj7W/DQmc80\n95xbqThzMzPLI99+y8zMLF+cuZmZ5Y3kWQFK3QEzM2sBLkuamZnlizM3M7M8clnSzMzyRS5LlroD\n1jokvSxpSD3LhqQLL+vbdpSkX7ZY58zMiszBLQckvSNplzptR0l6vPZ1RGwWEY+0eucaULePbZ2k\nndJEip9I+ihNwNin8S1BUn9JIemzgscLRejTeZL+tLL7KRZJX5Z0m6RZkuakiSpPLZivq6WO2+gX\nMEk/kDRB0kJJo1qyP21C7YjJYjzaIQc3K1vKrMj/A5OAPYDVgXWAN8gmV1wRPSKia3pssYLbFl26\nCW2x9rUhMI5sapKvRkR34CBgELBasY6zEt4HfglcX+qOtLjaKW9KeG/JUmufvbYVVpjdSeqcvul+\nLGkSsG2ddbeS9KykTyXdCnSqs3wvSc+nDOZJSZvXOc5p6Rv7nDQ1/DLbN7G/35P0SurDW5KOK1g2\nUdLeBa9XSZnCVun19qlfn0h6obAcK+kRSSMkPQHMAzZIGeRb6VhvSzp0eX2KiA8jYkrBDVurgY1W\n9L3V836PTu/3Y0n3S1qvYNkVkqZImivpGUnfSO1DgZ8BhxRmgnUz+cLsriCDPEbSe8A/m/CZNenz\nAc4HnoyIU9NNcImI1yLi0Ij4JO1rn1Qi/yT9W3yl4DghaaOC10uzMaXSuaSfSJohabqk76Vlw4FD\ngdPT53D38joXEX+JiL/RyN3kLR8c3MrTucCG6bEbX9xtG0kdgL8BNwI9gduAAwqWb0X2zfc4sskC\n/wDcJaljwf4PJpsefn1gc+CoZvRxBtlUGd2A7wGXSdo6LbsBOKxg3T2A6RHxXCoT/oPsG3pP4DTg\nDklrFax/ODCcLJuYCVwJ7B4RqwE7AM+n97pu+iO8bsH7X1fSJ8D8tO+Lm/HeliFpX7IgtT+wFvAY\ncHPBKk8DW6b382fgNkmdIuI+4ALg1mZkgv8NfAXYraHPTNKq1PP5LMcuZLMk1/c+v5ze14/S+7wH\nuDv9zjXFl8imOukDHAP8XtLqadLMm4CL0+ewdzreVZKuauK+c6b0swKUWvvstS3P39If4k/SH9+G\n/qc+GBgREbMjYgrZH69a2wOrAJdHxOKIuJ3sj2ut4cAfImJcRFRHxGhgYdqu1pUR8X5EzAbuJvvD\nvEIi4h8R8WZk/g08AHwjLf4TsIekbun14WTBGLKgd09E3BMRNRHxIDCBLADWGhURL0fEErK7ktcA\nAyV1jojpEfFy6sN7EdEjIt4r6Nd7EdEDWBP4OfDqCr61WQX/TqeltuOBCyPildSnC4Ata7O3iPhT\nRHwUEUsi4tdkd1rfeAWPW9d5EfF5RMyn8c9suZ/PcqwBTG/gmIcA/4iIByNiMXAp2d3fd2hinxcD\n/5N+L+8BPqOBzyEiToyIE5u47/zxOTfLif3SH+Ie6Y9vQ/9Tr0Oasj15t86yaXXmSipcvh7wkzqB\ntF/artYHBc/nAV1X5I0ASNpd0lhJs9Mx9iALKETE+8ATwAGSegC788Usv+sBB9Xp39eB3gW7X/re\nI+Jzsj+6xwPTJf1D0iaN9S8F7tHAnVqx81ZrFvw7XVrQ5ysK+jub7KxJn/RZnJZKlnPS8u61n8VK\nKPz3r/czW8HP5yOW/ZzrWoeC36WIqEn9aNKgHOCjFPxrNet3y8qDg1t5ms6yM+GuW2dZH2mZr2uF\ny6eQZX09Ch5dIqKwjLZSUonzDrJv9r1SsL6H7A9+rdFkGcdBwFMRUTtj7xTgxjr9WzUiLirYdpl5\nniLi/oj4Ftkf5leBa5vY1SpgbbLS6cqYAhxXp8+dI+LJdH7tdLJse/X0Wczhi89ieXNWfQ50KXj9\npeWsU7hdg5/ZCnw+D1FQwl6O98kCKZAN6CH7Paz9t5vXhH7Xx3N31eWypJWhMcBZklaX1BconPjp\nKbJS3cnKBmrsDwwuWH4tcLyk7ZRZVdKekpo7Gk6SOhU+gA5kpbeZwBJJuwO71tnub8DWwClk5+Bq\n/QnYW9JukirTPoek97m8g/eStG86t7SQrNRVU8+6+0vaWFJFOof3G+C5lMXVDtx4pBmfwTVk/x6b\npf10l3RQWrYa2b/HTKBK0i9YNph+CPTXsqM+nyeb6XgVSduQzVzckHo/sxX5fMjO5e4g6RJJX0rv\nZSNJf0oZ9hhgT0k7S1oF+Ena55MF/f5u6sNQsvOCTfUhsEFDK0iqSr9flUDt+8zvjSxclrQydD5Z\neehtsnNZteeriIhFZAMbjiIrjx0C/KVg+QTgWOB3wMfAZJo3YKTWDmSDM+o+Tib7Y/gx8F2yaeaX\nSueK7iAbtFLYvylA7QCNmWRZyU+p/3e9AjiVLKuYTfYH9QRYOnjks4IBJX2A+4BPgZfI/sh/u2Bf\n/cjKpSskIv4K/Aq4RdJcYCJZqRXg/nTM18n+zRawbEnxtvTzI0nPpufnkA0W+pjs3/rPjRy/oc+s\n3s9nOft5E/gvoD/wsqQ5ZP9GE4BPI+I1smz7t8AsYG9g7/Q7B9kXlb2BT8hGP/6toX7XcR2waSqr\n/g1A0jWSrilY5+dkv1tnpn7MT22WQ56J29qtlMV8OSIOa3TlViDpeWDniPBQcyupitX7R8chxYvb\nC/52rGfiNmsNknqSDQc/vNR9qRURKzwq1KzFtNNyYrG4LGntjqRjyUpn90bEo6Xuj5m1Pc7crN2J\niGtp+ohGs7KkMs/cHNzMzHJGOLi5LGlmZrnjzK2ZKjt3j6pua5e6G9ZODOzbvdRdsHbi3XffYdas\nWSuXdollb3lQhhzcmqmq29qs893LS90NayeeuGTPUnfB2omvbVeMEfdyWbLUHTAzMys2Z25mZjlU\n7pmbg5uZWQ6Ve3BzWdLMzHLHmZuZWQ6Ve+bm4GZmlje+FMBlSTMzyx9nbmZmOSNf5+bgZmaWR+Ue\n3FyWNDOz3HHmZmaWQ+WeuTm4mZnlULkHN5clzcwsd5y5mZnlja9zc3AzM8sjlyXNzMxyxpmbmVnO\n+CJuBzczs1wq9+DmsqSZmeWOMzczszwq78TNwc3MLHfksqTLkmZmljvO3MzMcqjcMzcHNzOzHCr3\n4OaypJmZ5Y4zNzOznPFF3M7czMzySUV8NHYo6XpJMyRNLGjrKelBSW+kn6sXLDtL0mRJr0naraB9\nkKSX0rIrlSK0pI6Sbk3t4yT1b6xPDm5mZrayRgFD67SdCTwcEQOAh9NrJG0KDAM2S9tcJakybXM1\ncCwwID1q93kM8HFEbARcBvyqsQ45uJmZ5U26zq1Yj8ZExKPA7DrN+wKj0/PRwH4F7bdExMKIeBuY\nDAyW1BvoFhFjIyKAG+psU7uv24Gd1UjHfM7NzCyH2sA5t14RMT09/wDolZ73AcYWrDc1tS1Oz+u2\n124zBSAilkiaA6wBzKrv4A5uZmbWmDUlTSh4PTIiRjZ144gISdEC/aqXg5uZWQ4VOXObFRHbrOA2\nH0rqHRHTU8lxRmqfBvQrWK9vapuWntdtL9xmqqQqoDvwUUMH9zk3M7M8asXRkvW4CzgyPT8SuLOg\nfVgaAbk+2cCR8amEOVfS9ul82hF1tqnd14HAP9N5uXo5czMzs5Ui6WZgCFn5cipwLnARMEbSMcC7\nwMEAEfGypDHAJGAJcFJEVKddnUg28rIzcG96AFwH3ChpMtnAlWGN9cnBzcwsh1pzQElEfKeeRTvX\ns/4IYMRy2icAA5fTvgA4aEX65OBmZpYzTR3Cn2c+52ZmZrnjzM3MLIfKPXNzcDMzy6FyD24uS5qZ\nWe44czMzy6PyTtwc3MzM8shlSTMzs5xx5mZmljdy5ubgZmaWMwLKPLa5LGlmZvnjzM3MLHd8+y0H\nNzOzHCrz2OaypJmZ5Y8zNzOzHHJZ0szM8kUuS7osaWZmuePMzcwsZwRUVJR36ubMzczMcseZm5lZ\nDpX7OTcHNzOzHCr30ZIuS5qZWe44czMzyxtfCuDgZmaWN9msAOUd3VyWNDOz3HHmZg06asf+DNt+\nXSS45an3+OOj7wBw5Df6c/jX1qM6gn9NmsFFd7/Kvluvw/BvbrB02016d2OvXz/OK+/PZdTwbVm7\nWycqK8XTb83mF7dPpCZK9KasVS1YsIBddtqRRQsXsqR6Cd/e/0DOOfd8Xnj+eX540vEsXLCAqqoq\nLv/tVWw7eHCpu5sTnhXAwc3q9eUvdWXY9uuy32WPs7g6GHXcYP45aQa9e3Rml4G92OOSx1hUXcMa\nXTsAcOez73Pns+8DsHHv1fjD0YN45f25APxg9HN8tnAJAFcdtTV7bNmbvz83vTRvzFpVx44due/B\nf9K1a1cWL17MN//76+y62+787/m/4OxzzmW3obtz3733cPZZp/PAw4+Uuru5UeaxzcHN6rdRr648\n/+4nLFhcA8D4yR8xdPMv8dV+3bnm4cksqs7aP/ps0X9su/dW6ywTvGoDW1WF6FBVQThrKxuS6Nq1\nKwCLFy9myeLFSFlmMXdu9uVnzpw59F5nnVJ203LGwc3q9dr0zzhtj43p0WUVFiyuZsima/PSlDms\nv9aqbLtBT07bY2MWLq7hgrte4cUpc5bZdq+tejP8ugnLtI0+bjBbrNuDR16Zwb0vOGsrJ9XV1eww\neBBvvjmZ4044icHbbcclv76cvffcjbPOOI2amhr+9eiTpe5mrpR7WdIDSqxeb874jGv++RY3HL8d\no48bzKRpc6muCSorKujRpQPfvvxJLrz7FX535NbLbLfluj2Yv6ia1z/4bJn2I/8wnsHnPkSHqgp2\nGLBma74VK7HKykrGPfM8k9+ZyoSnx/PyxImM/MPVXHzpZUx+ewoXX3oZJww/ptTdzI90KUCxHu1R\niwU3SSv8NUzSO5LuKHh9oKRRRe1Y4304T9JprXnMtmzMuCns85vHOeR3Y5kzbzFvz/ycDz6Zz30v\nfgDAC+/NoSaCnqt2WLrNXlv35u7n3l/u/hYtqeGhiR/yrYG9WqX/1rb06NGD/x6yEw88cB833Tia\n/b69PwAHHHgQE54eX+LeWZ60WHCLiB2auekgSZs2Z0NJLrMWWe1gkXV6dGLo5l/izmem8cDED/mv\njdYAYP21VmWVygpmf56dd5Ngzy3WWSa4delQyVrdOgJQWSF22nRt3pzxGVYeZs6cySeffALA/Pnz\nefihB9l4403ovc46PPbovwF45F//ZKONBpSym7lSe51bsR7tUYsFA0mfRURXSb2BW4Fu6XgnRMRj\nDWz6a+Bs4NA6++sJXA9sAMwDhkfEi5LOAzZM7e9Juh/YD1gVGABcCnQADgcWAntExGxJxwLD07LJ\nwOERMa8obz5Hrv7eIHp0WYUl1cEv7pjIpwuWcNu4KVw8bAvuO31HFlfXcNqfX1i6/uANejL9k/lM\n+Wj+0rYuHSq59pht6FhVgSTGTv6Im558rxRvx0rgg+nTOfboI6murqYmajjgwIPZY8+96N69Bz89\n9RSWLFlCx06d+N3VI0vd1VxppzGpaFoj0/kucH9EjJBUCXRpZP0xwImSNqrTfj7wXETsJ+mbwA3A\nlmnZpsDXI2K+pKOAgcBWQCeywHVGRGwl6TLgCOBy4C8RcS2ApF8CxwC/bahjkoaTBUQqV1ur8Xee\nAwf/9qn/aFtcHfz4pueXu/64N2ez/xXLVqRnfbaI/S57okX6Z23fVzffnLETnvuP9q99/es8Of6Z\nEvTIykFrBLengeslrQL8LSKW/1fxC9XAJcBZwL0F7V8HDgCIiH9KWkNSt7TsroiYX7DuvyLiU+BT\nSXOAu1P7S8Dm6fnAFNR6AF2B+xt7IxExEhgJ0LHXAA9mN7M2q72WE4ulxUdLRsSjwI7ANGCUpCOa\nsNmNaZt+TTzM53VeLyx4XlPwuoYvAvoo4AcR8VWyrLBTE49lZtbmebRkC5O0HvBhKgH+H7B1I5sQ\nEYuBy4AfFzQ/RjoPJ2kIMCsi5q5E11YDpqeM8tDGVjYzs/ajNcqSQ4CfSloMfEZ2zqsprgN+XvD6\nPLLy5otkA0qOXMl+nQOMA2amn6ut5P7MzNoGuSzZYsEtIrqmn6OB0U3cpn/B84XAOgWvZ5ONgqy7\nzXl1Xo8iKzkub59Ll0XE1cDVje3PzKy9yS4FKHUvSst3KDEzs9wpyUXPksYBHes0Hx4RL5WiP2Zm\n+dJ+L74ulpIEt4jYrhTHNTMrF2Ue21yWNDOz/PG9GM3McshlSTMzy5d2fPF1sbgsaWZmuePMzcws\nZ2qnvClnDm5mZjlU7sHNZUkzM8sdZ25mZjlU5ombg5uZWR65LGlmZrYSJP1Y0suSJkq6WVInST0l\nPSjpjfRz9YL1z5I0WdJrknYraB8k6aW07EqtRIR2cDMzy5siTlTaWHiR1Ac4GdgmIgYClcAw4Ezg\n4YgYADycXiNp07R8M2AocJWkyrS7q4FjgQHpMbS5H4GDm5lZzijdOLlYjyaoAjpLqgK6AO8D+/LF\ndGej+WLKsn2BWyJiYUS8DUwGBkvqDXSLiLEREcANLGeas6ZycDMzs2aLiGnApcB7wHRgTkQ8APSK\niOlptQ+AXul5H2BKwS6mprY+6Xnd9mZxcDMzy6EilyXXlDSh4DH8i+NodbJsbH2yCaZXlXRYYV9S\nJhat9+49WtLMLJcqijtaclZEbFPPsl2AtyNiJoCkvwA7AB9K6h0R01PJcUZafxrQr2D7vqltWnpe\nt71ZnLmZmdnKeA/YXlKXNLpxZ+AV4C7gyLTOkcCd6fldwDBJHSWtTzZwZHwqYc6VtH3azxEF26ww\nZ25mZjnUWpe5RcQ4SbcDzwJLgOeAkUBXYIykY4B3gYPT+i9LGgNMSuufFBHVaXcnAqOAzsC96dEs\nDm5mZjmTnStrvYu4I+Jc4Nw6zQvJsrjlrT8CGLGc9gnAwGL0yWVJMzPLHWduZmY5VFHed99ycDMz\nyyPfW9LMzCxnnLmZmeVQmSduDm5mZnkjsvtLljOXJc3MLHecuZmZ5ZBHS5qZWb40faqa3HJZ0szM\ncseZm5lZDpV54ubgZmaWN6LoU960Oy5LmplZ7jhzMzPLoTJP3BzczMzyyKMlzczMcsaZm5lZzmST\nlZa6F6Xl4GZmlkMeLWlmZpYz9WZukro1tGFEzC1+d8zMrBjKO29ruCz5MhAs+xnVvg5g3Rbsl5mZ\nrYRyHy1Zb3CLiH6t2REzM7NiadI5N0nDJP0sPe8raVDLdsvMzJoru/1W8R7tUaPBTdLvgJ2Aw1PT\nPOCaluyUmZmthDTlTbEe7VFTLgXYISK2lvQcQETMltShhftlZmbWbE0JboslVZANIkHSGkBNi/bK\nzMxWSjtNuIqmKcHt98AdwFqSzgcOBs5v0V6ZmdlKaa/lxGJpNLhFxA2SngF2SU0HRcTElu2WmZlZ\n8zX19luVwGKy0qTvamJm1obVjpYsZ00ZLXk2cDOwDtAX+LOks1q6Y2Zm1nweLdm4I4CtImIegKQR\nwHPAhS3ZMTMzs+ZqSnCbXme9qtRmZmZtVPvMt4qnoRsnX0Z2jm028LKk+9PrXYGnW6d7Zma2oiRP\nedNQ5lY7IvJl4B8F7WNbrjtmZmYrr6EbJ1/Xmh0xM7PiKfPErfFzbpI2BEYAmwKdatsj4sst2C8z\nM7Nma8o1a6OAP5Kdn9wdGAPc2oJ9MjOzlVTulwI0Jbh1iYj7ASLizYj4OVmQMzOzNkoq3qM9asql\nAAvTjZPflHQ8MA1YrWW7ZWZm1nxNCW4/BlYFTiY799YdOLolO2VmZs0n5EsBGlshIsalp5/yxYSl\nZmbWVrXjcmKxNHQR919Jc7gtT0Ts3yI9MjMzW0kNZW6/a7VetEMD+3bniUv2LHU3rJ1YfdsflLoL\n1k4sfO29ouynvY5yLJaGLuJ+uDU7YmZmxVPuc5OV+/s3M7McaupkpWZm1k4IlyWbHNwkdYyIhS3Z\nGTMzKw7PxN0ISYMlvQS8kV5vIem3Ld4zMzOzZmrKObcrgb2AjwAi4gVgp5bslJmZrZwKFe/RHjWl\nLFkREe/Wqd9Wt1B/zMxsJWX3hGynUalImhLcpkgaDISkSuCHwOst2y0zM7Pma0pZ8gTgVGBd4ENg\n+9RmZmZtVGuWJSX1kHS7pFclvSLpvyT1lPSgpDfSz9UL1j9L0mRJr0naraB9kKSX0rIrtRLpZ6PB\nLSJmRMSwiFgzPYZFxKzmHtDMzFpeK095cwVwX0RsAmwBvAKcCTwcEQOAh9NrJG0KDAM2A4YCV6Wq\nIMDVwLHAgPQY2tz335SZuK9lOfeYjIjhzT2omZnlg6TuwI7AUQARsQhYJGlfYEhabTTwCHAGsC9w\nS7q07G1Jk4HBkt4BukXE2LTfG4D9gHub06+mnHN7qOB5J+DbwJTmHMzMzFqeoDWnvFkfmAn8UdIW\nwDPAKUCviJie1vkA6JWe9wHGFmw/NbUtTs/rtjdLU6a8ubXwtaQbgcebe0AzM2t5Rb634pqSJhS8\nHhkRI9PzKmBr4IcRMU7SFaQSZK2ICEn1zjLTEppz+631+SICm5lZ/s2KiG3qWTYVmFow9+ftZMHt\nQ0m9I2K6pN7AjLR8GtCvYPu+qW1ael63vVmacoeSjyXNTo9PgAeBs5p7QDMza3mtNaAkIj4gu2Rs\n49S0MzAJuAs4MrUdCdyZnt8FDJPUUdL6ZANHxqcS5lxJ26dRkkcUbLPCGszc0gG24IvoWRMRrZpa\nmpnZipHUmufcILv++SZJHYC3gO+RJU9jJB0DvAscDBARL0saQxYAlwAnRUTtjUFOBEYBnckGkjRr\nMAk0EtxSnfSeiBjY3AOYmVm+RcTzwPLKljvXs/4IYMRy2icARYk3TTnn+LykrYpxMDMzax2tfJ1b\nm1Nv5iapKiKWAFsBT0t6E/icbJRpRMTWrdRHMzNbQe31hsfF0lBZcjzZ8M59WqkvZmZmRdFQcBNA\nRLzZSn0xM7MiaOWLuNukhoLbWpJOrW9hRPymBfpjZmZFUOaxrcHgVgl0JWVwZmZm7UVDwW16RPxP\nq/XEzMyKox3PoF0sjZ5zMzOz9kdl/ie8oevclnvxnZmZWVtXb+YWEbNbsyNmZlYc2WjJUveitJoz\nK4CZmbVx5R7cijzlj5mZWek5czMzyyGV+YVuDm5mZjnjc24uS5qZWQ45czMzy5t2PFVNsTi4mZnl\nULnfONllSTMzyx1nbmZmOeMBJQ5uZma5VOZVSZclzcwsf5y5mZnljqgo81kBHNzMzHJGuCzpsqSZ\nmeWOMzczs7zxTNwObmZmeeSLuM3MzHLGmZuZWc54QImDm5lZLrksaWZmljPO3MzMcqjMEzcHNzOz\nvBEuy5X7+zczsxxy5mZmljcClXld0sHNzCyHyju0uSxpZmY55MzNzCxnspm4yzt3c3AzM8uh8g5t\nLkuamVkOOXMzM8uhMq9KOriZmeWPyv5SAJclzcwsd5y5mZnljG+/5eBmZpZLLkuamZnljDM3K4oH\n7r+P0049herqao46+vv89PQzS90lawXXnHsou+84kJmzP2Wbgy4A4IIf7cceOw5k0eJq3p46i+Hn\n/ok5n82nqqqCq39xKFtu0o+qygpu+sd4Lr3+AQDu/N2JfGmtblRVVvLEc2/yowtvpaYm+P6BX+e4\ng3ekuqaGz+ct5KRf3syrb31QyrfcbpR33ubMzYqgurqaH518EnfefS/PvTiJ2265mVcmTSp1t6wV\n3Hj3WPY96ffLtD089lUGHXQBgw+5kDfencFPj94VgAN22ZqOHarY9uAL2OHQX/H9A77Gur17AnDY\nGdez3SEXMejAEay1elcO+NbWANx67wS2PfgCth92Eb8Z/RC/OnX/1n2D7VW6cXKxHu2Rg5uttKfH\nj2fDDTdi/Q02oEOHDhx0yDD+fvedpe6WtYInnn2T2XPmLdP28NhXqa6uAWD8S2/Tp1cPAIKgS6cO\nVFZW0LlSaB9dAAAWRklEQVRjBxYtrubTzxcALP1ZVVXBKlWVRMQy7QCrdu5AEC3+niwfXJa0lfb+\n+9Po27ff0td9+vRl/PhxJeyRtRVH7Ptf3P7AswD85aHn2GvI5rz94Ai6dOrA6Zf+hY/nfhEY7/r9\nSWwzcD0eeGISf3nouaXtxx28IycfthMdVqli6HFXtvp7aI88WtLv38xayOnH7EZ1dQ233PM0ANtu\n1p/q6ho22PVsvrLnuZxy+Dfp32eNpevvc9LvWf9bP6NjhyqGbLvx0vY/jHmUzfY5n59fcSdnfn9o\nq7+P9splyVYi6clmbrelpJA0tKCth6QTC173l/TdlejbI5K2ae725W6ddfowdeqUpa+nTZtKnz59\nStgjK7XD9t6OPXYcyFFnj1radvDu2/DAk5NYsqSGmR9/xlPPv8WgTdddZruFi5Zw9yMvsveQr/7H\nPsfc/wx7D9m8pbtuOdFqwS0idmjmpt8BHk8/a/UATix43R9odnCzlbPNttsyefIbvPP22yxatIjb\nbr2FPffap9TdshL51g5f4dSjduHAH/2B+QsWL22f+sHspRlZl04dGLx5f15750NW7dyBL63ZDYDK\nygp2//pmvPbOhwBsuO5aS7ff/RubMXnKzFZ8J+2bivhoj1rtnJukzyKiq6TewK1At3T8EyLisXq2\nEXAQ8C3gMUmdImIBcBGwoaTngQeBbwBfSa9HA38FbgRWTbv6QUQ8mfZ5BnAYUAPcGxFnFhyvArge\nmBoRP19Of4YDwwH6rbtu3cVlq6qqisuu+B1777kb1dXVHHnU0Wy62Wal7pa1gtEXHsU3Bg1gzR5d\nmXzf//K/19zDT7+3Kx07VPH3q38AwPiX3uHkEbdwza2PMvL8w3jm9rOR4MY7xzLxjfdZu+dq3H75\ncXRYpYqKCvHohDe49vbHATjhkB3ZabtNWLykmk/mzuPYc24o5dttV1q7miipEpgATIuIvST1JPtb\n3x94Bzg4Ij5O654FHANUAydHxP2pfRAwCugM3AOcErWji1a0P83cbsUP9EVw+wnQKSJGpA+jS0R8\nWs82XwP+JyJ2lvRn4I6IuENSf+DvETEwrTcEOC0i9kqvuwA1EbFA0gDg5ojYRtLuwDnALhExT1LP\niJgt6RHgTOAUYGJEjGjs/QwatE08MW7CSn0mVj5W3/YHpe6CtRMLXxtDzbwZKxWaNtpsi/j1LfcX\nq0vst3nvZyKiwVM3kk4FtgG6peB2MTA7Ii6SdCawekScIWlT4GZgMLAO8BDw5YioljQeOBkYRxbc\nroyIe5vT51IMKHka+J6k84Cv1hfYku8At6Tnt7BsabIhqwDXSnoJuA3YNLXvAvwxIuYBRMTsgm3+\nQBMDm5lZW5aNllTRHo0eT+oL7An8X0HzvmSVNNLP/Qrab4mIhRHxNjAZGJyqet0iYmzK1m4o2GaF\ntXpwi4hHgR2BacAoSUcsb72U1R0A/ELSO8BvgaGSVmvCYX4MfAhsQfZNokMTtnkS2ElSpyasa2ZW\nTtaUNKHgMbzO8suB08lO99TqFRHT0/MPgF7peR9gSsF6U1Nbn/S8bnuztHpwk7Qe8GFEXEsW5beu\nZ9WdgRcjol9E9I+I9YA7gG8DnwKFQa7u6+7A9IioAQ4HKlP7g2RZY5fUl54F21xHlgaPkeTr/8ys\nXZOK9wBmRcQ2BY+RXxxHewEzIuKZ+vqSMrFWvQK/FGXJIcALkp4DDgGuqGe975ANDCl0B/CdiPgI\neELSREmXAC8C1ZJekPRj4CrgSEkvAJsAnwNExH3AXcCENPjktMKdR8RvgOeAG9PgEjOzdkhF/a8R\nXwP2SRW2W4BvSvoT8GEqNZJ+zkjrTwP6FWzfN7VNS8/rtjfvE2itASV54wEltiI8oMSaqhgDSgZs\ntmVcfusDxeoSe321V6MDSmDZwX0p8fioYEBJz4g4XdJmwJ/5YkDJw8CAegaU/DYi7mlOn11+MzPL\noTZwY5GLyE7zHAO8CxwMEBEvSxoDTAKWACdFRHXa5kS+uBTg3vRoljYR3CSNAzrWaT48Il4qRX/M\nzNqz2tGSrS0iHgEeSc8/Ihs7sbz1RgD/MTI9IiYAA4vRlzYR3CJiu1L3wczM8qNNBDczMysitYmy\nZEk5uJmZ5VC5BzcPdzczs9xx5mZmlkNNuD4t1xzczMxyRkBFecc2lyXNzCx/nLmZmeWQy5JmZpY7\nHi1pZmaWM87czMxyyGVJMzPLFY+WdFnSzMxyyJmbmVnuNGmS0VxzcDMzyxvfONllSTMzyx9nbmZm\nOVTmiZuDm5lZ3mSjJcs7vLksaWZmuePMzcwsh8o7b3NwMzPLpzKPbi5LmplZ7jhzMzPLIV/EbWZm\nuVPmgyVdljQzs/xx5mZmlkNlnrg5uJmZ5VKZRzeXJc3MLHecuZmZ5YzwaEkHNzOzvPGUNy5LmplZ\n/jhzMzPLoTJP3BzczMxyqcyjm8uSZmaWO87czMxyRx4tWeoOmJlZ8Xm0pJmZWc44czMzyxlR9uNJ\nHNzMzHKpzKOby5JmZpY7ztzMzHLIoyXNzCx3PFrSzMwsZ5y5mZnlUJknbg5uZma542sBXJY0M7P8\nceZmZpZDHi1pZma5Ijxa0mVJMzPLHWduZmY5VOaJm4ObmVkulXl0c1nSzMyaTVI/Sf+SNEnSy5JO\nSe09JT0o6Y30c/WCbc6SNFnSa5J2K2gfJOmltOxKqflnDh3czMxySEX8rxFLgJ9ExKbA9sBJkjYF\nzgQejogBwMPpNWnZMGAzYChwlaTKtK+rgWOBAekxtLnv38HNzCyHpOI9GhIR0yPi2fT8U+AVoA+w\nLzA6rTYa2C893xe4JSIWRsTbwGRgsKTeQLeIGBsRAdxQsM0Kc3AzM7OikNQf2AoYB/SKiOlp0QdA\nr/S8DzClYLOpqa1Pel63vVk8oMTMLIeKPJ5kTUkTCl6PjIiRyxxP6grcAfwoIuYWni6LiJAUxe1S\nwxzczMzyqLjRbVZEbFPvoaRVyALbTRHxl9T8oaTeETE9lRxnpPZpQL+Czfumtmnped32ZnFZ0szM\nmi2NaLwOeCUiflOw6C7gyPT8SODOgvZhkjpKWp9s4Mj4VMKcK2n7tM8jCrZZYc7czMxyJpsUoNUu\ndPsacDjwkqTnU9vPgIuAMZKOAd4FDgaIiJcljQEmkY20PCkiqtN2JwKjgM7AvenRLA5uZmZ504RR\njsUSEY9TfxF053q2GQGMWE77BGBgMfrlsqSZmeWOM7dmevbZZ2Z1XkXvlrofbcyawKxSd8LaDf++\nLN96xdhJmd99y8GtuSJirVL3oa2RNKGhEVVmhfz70sLKPLq5LGlmZrnjzM3MLHeadE/IXHNws2Ia\n2fgqZkv596UFeSZusyKpezses4b498VakjM3M7OcEWU/nsTBzcwsl8o8urksaWZmuePMzUpKUk9g\nzYh4vdR9sfZDktKEllaPch8t6czNSkZSJ+Bk4GhJXyl1f6ztk9QPsvnBSt2Xtq61ZuJuqxzcrGQi\nYgHwUHp5kKRNS9kfa3skdZXUIT3/CnCxpNVK3C1rBxzcrCTSfE21dxS/C+gGHOgAZ7UkrQrcBByU\nmualx2dpcsylv0f2n1TER3vk4GatrvZ8iaT1JVVFxJPAH4HuZAHOJUojIj4HbgW+J+kQoD8wPzKL\n0zouT9pyeUCJtboU2PYEzgEek/QZcDnZHSuOAQ6TdFNETCplP610JFVGRHVE/FnSTOAM4BlgfUlX\nAFOBhUBVndmfDVp1Pre2ypmbtTpJ2wMXAIeQfcHaD7gYmAmMBlYFFpWsg1ZSKbOvlvQtSRdHxIPA\nFWQTXy4C3ks/uwLjStjVNq68C5PO3KzVSKoAgmweryOATYAdgTOB4cClZN/Qz04lKStDKbPfGbgK\nOC613S1pCXAq8HpE3F3KPlrb58zNWlzBSf+u6XzJ3yPiBbKM7fsRcT8wg+zLVi8HtvKlTBUwFDgn\nIv5ZO1oyIu4FrgHOkNSnlP1s64QvBXBwsxZXcI7tYUnnSdo/LVobGC5pO2AwcGlETCxZR63k0pef\nJcACYHtJnSJiEYCkbYF7gH0iYlop+9kelHdR0sHNWoGk3sChZGXH2cBuKdgdDfQDfgFcGBEvlq6X\nViq1mb2kdSX1Tc33AqsA/52WbQFcBnw5ImaXpKPWrvicm7UoSdsAWwDTIuJWSWsBuwHfBlaJiL0k\ndYmIeb6lUnkqyOwvBJ6U1DMiDk6XhBwu6Qyyy0R+mcrZ1gTttZxYLA5u1mIkDSEb/Xg/2fD+myPi\nWUn3Ah2AfSWNj4j3wdcslZuC6x23JxstuxdZpna9pIciYhdJo8i+HM2JiDf9Bajpyv3ekg5u1iIk\nrQ/8DDg8Ih6VNBn4k6RDI+I5SXcC99UGNisf6Z6ii9Nw/17AR8DBwACy0ZHdgUckPRkROwDP1m7r\nwGZN5XNuVjQF5062JfsG3p1sRCQRcTFwHXCXpEER8ZEDW/lJl4PsAPxI0l5k51s/BSYBewLXR8Sn\nZBn/uul3yZqjzEeUOLhZ0aQS045kJaaXyC7U7iLpB2n5r4Hfk118a+XrRWBX4Ebg9oj4gOxP6HRg\nQ0nHkpUovxURT5eum+1bmcc2BzcrHkkbAycAoyLiGeAR4GFgE0k/AYiIiyLi377hbXmRtKqkvhFR\nA6yXmv8F7J6G+9eQzRAxjyywXRMRr5Sou5YDPudmxfRVoBewi6R7ImKmpPvIhnQPkbReRLwLPndS\nhvoDv5Q0ARgI/AT4mOz+or8BTgTeIgt4F0TEEg8eab72fPF1sThzs2YrOMfWV1L3iLid7I/VXLK7\n+6+Rzp/cDfyiNrBZ+YmIl4HJZIOMxqWL9WeS3WKro6SHyTL9xekibn8BWkkq4n/tkYObNYukinSO\nbXeyC26vk/Qo8Arwd6D2GqU1IuLTdF7FyoikHpK6FDRNBH4NHCFp54hYlC7cPxsYBfw4IsaWoKuW\nQy5L2gqR1Dki5kdEjaSNgP8FjouIJyVdCfyN7CLtVdLPVcmGelsZkdQTeB14SNJjEfH7iBidlk0B\nfiPpSOATYP/aaWtciiyi9plwFY2DmzWZpO7ARZL+GhEPkP1hepXsjxgRcbKkm4EzI+JcSU9HxPQS\ndtlK52PgAbIRkIdKGgw8DtwWEddKWgTcASwBflS7kQNb8ZR5bHNZ0lZIN7LzJt9NU5LMBdYAdilY\n5x7SXGwObOUrBalnyQYY7UhWdtwR+LekncgGjmwHHJDu9m9WVM7crFGSVkvnzaZIugEYRnbT45lk\nAwRGSdoEmJPaTy9db62tiIhLJd1D9uVnIrAlWaY/DNgIOMSzQLScch8t6eBmDZLUH7hd0jPAGOAN\n4I/AQrLh3L8CDgJ2B9YhGxTwkM+dlDdJlRFRTZaxfZvsjv7XpYC3NtlNs2eVso/51n5HORaLg5s1\nphPQG9gXeIfsDiPXAKsDT5IN/R8REVcUbuTAVt5SYAMYB5wHPBURl6a2mf79sJbmc25WrzTc/1Wy\nstIc4D3gEOB9sntHHpheX5yGffv3yZZK2fu7wKlA19rZsx3YWp5n4nbmZg1Iw/0rIuIVSYcBt5Dd\nPeI6SbeT3cV9X+D5iPikpJ21kiiYtqYi3UJrqYIgNhWo+c+tzVqOg5s1qCDAPS1pGHBzuhfg74HX\nyG6S7OuTylBBYNuZLDO7PyIW1F0vIiZKOiMippWgm1amXEayRhUGOLIy5DmSTqqzjgNbGUkDRkLS\nUOBq4OPlBTZlKiLiXUldJK3R+r0tT+VelnRws6UK7hX5H78XBQHuGWBv4OXW7p+VnqSN0qUh1ZJW\nJxtQdHyakPYbko5MF2zXqki/Oz3Irm3rWZKOl6Fyv7eky5IGNK3EVCeDcymyPPUC1pY0NiI+lvQv\n4Jg0B1sFsJjsXOx4SVXp7v7dgduAn0bEG6XrupUTZ27W5BJT7eppm85klwNYGYmIJ8gmon1LUjey\n69jGA7+NiEPIroXcTFKHFNhWB/4K/E9EPFqqfpedIpYkXZa0dmdFS0y1F+amEtMjZLfesjKTpjE6\nhew6x1kRcUW6cfY3yG6k/X8RsSit/h3glxHxWIm6W5aKOQt3O41tLkuWOZeYrFki4k5Ji4FnJA0C\nFpBd9/jziPhHbck6Iq4qbU+tXDm4lbGIeELSamQlps3JSkx7Ak+nb+L7AN9LJaZFKbu7AzjX38Qt\nIu6RVEM2h9/GwBkRsaDg/K3PyZZSe025isRlyTLnEpOtjIi4D/g+sFXtedragObAVloeLWllzyUm\nWxkR8Q/w6FlrWxzcDHCJyVaefz/alvY6yrFYXJa0pVxiMssPj5Y0K+ASk5nlgTM3Wy4HNrN2rhVT\nN0lDJb0mabKkM4v9VprDmZuZWQ611ihHSZVkkxh/i2x6o6cl3RURk1qlA/Vw5mZmZitjMDA5It5K\nlw3dQjbPY0k5czMzy5nambhbSR9gSsHrqcB2rXb0eji4We5Iqia7uW8V2aUNR0bEvGbuawhwWkTs\nle7YsmlEXFTPuj2A767o9YCSzgM+i4hLm9JeZ51RwN8j4vYmHqt/Wn/givTR2pdnn33m/s6raM0i\n7rKTpAkFr0dGxMgi7r/oHNwsj+ZHxJYAkm4Cjgd+U7swzVuniKhZkZ1GxF3AXQ2s0gM4EfDF7lZS\nETG0FQ83DehX8Lpvaispn3OzvHsM2EhS/zSa6wZgItBP0q6SnpL0rKTbJHWFpSO/XpX0LLB/7Y4k\nHSXpd+l5L0l/lfRCeuwAXARsKOl5SZek9X4q6WlJL0o6v2BfZ0t6XdLjZBfNN0jSsWk/L0i6Q1KX\ngsW7SJqQ9rdXWr9S0iUFxz5uZT9Is3o8DQyQtL6kDsAwGv4S2Coc3Cy3JFUBu5OVKCGb4eCqiNgM\n+Bz4ObBLRGwNTABOldQJuJZstvFBwJfq2f2VwL8jYgtga7KZyc8E3oyILSPip5J2TcccDGwJDJK0\nY7rF2bDUtgewbRPezl8iYtt0vFeAYwqW9U/H2BO4Jr2HY4A5EbFt2v+xktZvwnHMVkhELAF+ANxP\n9rs5JiJeLm2vXJa0fOos6fn0/DHgOmAd4N2IGJvatwc2BZ7IqpR0AJ4CNgHerp3OR9KfgOHLOcY3\ngSMAIqIamJNmTSi0a3o8l153JQt2qwF/rT0PKKkp33IHSvolWemzK9kfklpjUon1DUlvpfewK7C5\npAPTOt3TsV9vwrHMVkhE3APcU+p+FHJwszxaes6tVgpgnxc2AQ9GxHfqrLfMditJwIUR8Yc6x/hR\nM/Y1CtgvIl6QdBQwpGBZ3QvuIx37hxFRGARrB5SY5Z7LklauxgJfk7QRgKRVJX0ZeBXoL2nDtN53\n6tn+YeCEtG1lmsT1U7KsrNb9wNEF5/L6SFobeBTYT1LnNJ/e3k3o72rAdEmrAIfWWXaQpIrU5w2A\n19KxT0jrI+nLklZtwnHMcsGZm5WliJiZMqCbJXVMzT+PiNclDQf+IWkeWVlzteXs4hRgpKRjgGrg\nhIh4StITkiYC96bzbl8BnkqZ42fAYRHxrKRbgReAGWQn5BtzDjAOmJl+FvbpPWA80A04Ps3m8H9k\n5+KeTaNDZwL7Ne3TMWv/5FsImplZ3rgsaWZmuePgZmZmuePgZmZmuePgZmZmuePgZmZmuePgZmZm\nuePgZmZmuePgZmZmufP/o5SAIQ3qu04AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbd3dc284e0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value, pred_value = Train.pred_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:20.431436Z",
     "start_time": "2017-06-16T20:42:20.162511Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[2114   38]\n",
      " [   0 9698]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbUAAAGgCAYAAAAtsfn1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmcXfP9x/HXO4skkpBYakmQEEuD2oLU0mopQSxVImqr\nvZaWqrW0tL+mVVSrVVpLm6BFUETttFq1JmKJpSGkSASJSCwJJjOf3x/nO3qNzGQmuTP33nPeT4/z\nmHv2z70Z87mf7/d7zlFEYGZmlgedKh2AmZlZuTipmZlZbjipmZlZbjipmZlZbjipmZlZbjipmZlZ\nbjipmZlZbjipmZlZbjipmZlZbnSpdABmZta+Oi+zRsSC+WU7XsyfeVdEDCvbAcvISc3MLOdiwXy6\nrTuibMf78MnfrVC2g5WZk5qZWe4JVIzepmK8SzMzKwRXamZmeSdAqnQUHcJJzcysCNz8aGZmVltc\nqZmZFYGbH83MLB88+tHMzKzmuFIzMysCNz+amVkuCDc/mpmZ1RpXamZmuSc3P5qZWY64+dHMzKy2\nuFIzMysCNz+amVk++OJrMzOzmuNKzcws7/zoGTMzyxU3P5qZmdUWV2pmZrlXnIEiTmpmZkXQqRh9\nasVI3WZmVgiu1MzM8s536TczM6s9rtTMzIrA16mZmVk+FGf0YzHepZmZFYIrNTOzInDzo5mZ5Yab\nH83MzGqLKzUzs7yT3PxoZmY54uZHMzOz2uKkVjCSnpW0XTPrtpM0rYV9R0v6absFZ2btp7EJshxT\nFXNSyxFJ/5W0Q5Nl35L078b5iFg/Iu7v8OBa0DTGaifpK5ImSZoj6W1JN0nq18p9B0gKSe+XTE+V\nIaazJV29pMcpF0nrSLpe0ixJcyU9LelESZ3b+byL/OIl6ThJEyR9JGl0e8ZTPdLF1+Waqlh1R2fW\nAZRpy/8LzwG7AH2BVYEXgUvaeNo+EdErTRu1cd+yk1S2/nVJawGPAq8BG0bEssA+wGZA73KdZwm8\nDvwU+GOlA7Hyc1IrmNJqTlKP9M32HUnPAZs32XYTSRMlvSfpOqB7k/XDJT2ZKpaHJH2hyXlOSt/Q\n50q6TtKn9m9lvIdIej7F8LKko0rWPSNpt5L5rqky2CTND01xzZH0VGmzq6T7JY2S9CAwD1gzVYwv\np3NNlbT/wmKKiDcj4rWIiLSoHhjU1vfWzPs9NL3fdyTdJWmNknUXSnpN0ruSHpe0bVo+DPgBsG9p\n5de0ci+t5koqxsMkvQr8vRWfWas+H+DHwEMRcWJEzEif2eSI2D8i5qRj7Z6awuekf4vPl5wnJA0q\nmf+k+lJqIpf0fUlvSZoh6ZC07khgf+CU9DncurDgIuKvEXEz8Har/lHyws2PVgBnAWulaSfg4MYV\nkpYCbgauApYDrge+UbJ+E7JvukcBywN/AMZJ6lZy/BHAMGAg8AXgW4sR41vAcGAZ4BDgV5I2Teuu\nBA4o2XYXYEZEPJGaA28j+0a+HHAScKOkFUu2PxA4kqx6mAn8Btg5InoDWwFPpve6evrju3rJ+19d\n0hxgfjr2uYvx3j5F0h5kyWkvYEXgAeCakk3GAxun9/MX4HpJ3SPiTuBnwHWLUfl9Gfg8sFNLn5mk\nnjTz+SzEDsANLbzPddL7OiG9z9uBW9PvXGusDCwL9AMOA34nqW9EXAr8GTg3fQ67pfNdLOniVh47\nnxofPePmR6tBN6c/wHPSH92W/mceAYyKiNkR8RrZH61GQ4GuwK8joi4ibiD7o9roSOAPEfFoRNRH\nxBjgo7Rfo99ExOsRMRu4lewPcptExG0R8VJk/gncDWybVl8N7CJpmTR/IFkShizZ3R4Rt0dEQ0Tc\nA0wgS3yNRkfEsxGxAFgANAAbSOoRETMi4tkUw6sR0SciXi2J69WI6AOsAJwJ/KeNb21Wyb/TSWnZ\nt4GfR8TzKaafARs3VmsRcXVEvB0RCyLil0A3YN02nrepsyPig4iYz6I/s4V+PguxPDCjhXPuC9wW\nEfdERB1wPtCDLFG2Rh3wk/R7eTvwPi18DhFxTEQc08pjW41zUsufPdMf4D7pj25L/zOvStbv0eiV\nJuumlzSxNV2/BvD9Jgl0tbRfozdKXs8DerXljQBI2lnSI5Jmp3PsQpZIiIjXgQeBb0jqA+xM9k29\nMb59msS3DbBKyeE/ee8R8QHZH9tvAzMk3SZpvUXFlxL2GOAWta1faoWSf6fzS2K+sCTe2WTfsful\nz+Kk1DQ5N61ftvGzWAKl//7NfmZt/Hze5tOfc1OrUvK7FBENKY5WDbYB3k5Jv9Fi/W4ViweKWDHM\nIEtEjVZvsq6f9KkG9NL1r5FVeX1KpqUjorS5bImkpswbyb7Jr5SS9O1kf+gbjSGrMPYBHo6I6SXx\nXdUkvp4RcU7JvqUJm4i4KyK+RvYH+T/AZa0MtQvwObIm0iXxGnBUk5h7RMRDqf/sFLLqum/6LOby\nv88iFnK8D4ClS+ZXXsg2pfu1+Jm14fO5l5Km6oV4nSyBAtlAHbLfw8Z/u3mtiLs5C/scDNynZoUw\nFjhdUl9J/YHvlKx7mKxJ7rvKBmDsBWxRsv4y4NuStlSmp6RdJS3u6DZJ6l46AUuRNbHNBBZI2hnY\nscl+NwObAseT9bE1uhrYTdJOkjqnY26X3ufCTr6SpD1S39FHZE1aDc1su5ekdSV1Sn10FwBPpKqt\ncUDG/YvxGfye7N9j/XScZSXtk9b1Jvv3mAl0kfQjPp1E3wQG6NOjOJ8ERqZ/vyHA3os4f7OfWVs+\nH7K+2q0knSdp5fReBkm6OlXUY4FdJW0vqSvw/XTMh0ri/maKYRhZv19rvQms2dIGkrqk36/OQOP7\n9N2VcsJJrdh+TNYMNJWsr6qxP4qI+JhswMK3yJrB9gX+WrJ+AnAEcBHwDjCFxRsI0mgrskEXTafv\nkv0RfAf4JjCudKfUF3Qj2WCU0vheAxoHXswkq0JOpvnf+U7AiWRVxGyyP6RHwyeDQt4vGSjSD7gT\neA+YRPbH/eslx1qNrFm0TSLiJuAXwLWS3gWeIWtSBbgrnfMFsn+zD/l00+H16efbkiam1z8kGwT0\nDtm/9V8Wcf6WPrNmP5+FHOcl4IvAAOBZSXPJ/o0mAO9FxGSy6vq3wCxgN2C39DsH2ReU3YA5ZKMZ\nb24p7iauAAan5tObAST9XtLvS7Y5k+x367QUx/y0LN8K0vyoT3eZmNWeVLWsExEHLHLjDiDpSWD7\niCjWkHGrWp36rBHdtjujbMf78JajHo+IIWU7YBm55LaaJmk5smHdB1Y6lkYR0eZRnmZWHtVdR5q1\nQNIRZE1kd0TEvyodj1nVUnFGP7pSs5oVEZfR+hGKZsVW5aMWy6W6U66ZmVkbuFIzMysAFaRSc1Jb\nTH2WWz5W7b/6ojc0K9G9a7s+ecVy6NVX/susWbOWKCMJJzVbhFX7r85V4/5Z6TCsxgzuVw1PXrFa\nsvXQzRe9kX3CSc3MLO/Ep28ul2NOamZmuafCND969KOZmeWGKzUzswIoSqXmpGZmVgBFSWpufjQz\ns9xwpWZmVgBFqdSc1MzM8q5AQ/rd/GhmZrnhSs3MLOfk69TMzCxPJJVtasW5vifpWUnPSLpGUndJ\ny0m6R9KL6Wffku1PlzRF0mRJO5Us30zSpLTuN2rFyZ3UzMysbCT1A74LDImIDYDOwEjgNOC+iFgb\nuC/NI2lwWr8+MAy4WFLjnb8vAY4A1k7TsEWd30nNzKwAOrJSI+va6iGpC7A08DqwBzAmrR8D7Jle\n7wFcGxEfRcRUYAqwhaRVgGUi4pGICODKkn1aPLGZmeVcR/WpRcR0SecDrwLzgbsj4m5JK0XEjLTZ\nG8BK6XU/4JGSQ0xLy+rS66bLW+RKzczM2moFSRNKpiMbV6S+sj2AgcCqQE9JB5TunCqvaI/AXKmZ\nmeVd+a9TmxURQ5pZtwMwNSJmAkj6K7AV8KakVSJiRmpafCttPx1YrWT//mnZ9PS66fIWuVIzMyuA\nDuxTexUYKmnpNFpxe+B5YBxwcNrmYOCW9HocMFJSN0kDyQaEPJaaKt+VNDQd56CSfZrlSs3MzMom\nIh6VdAMwEVgAPAFcCvQCxko6DHgFGJG2f1bSWOC5tP2xEVGfDncMMBroAdyRphY5qZmZ5VxHX3wd\nEWcBZzVZ/BFZ1baw7UcBoxayfAKwQVvO7aRmZlYAvqOImZlZjXGlZmZWBMUo1JzUzMxyT25+NDMz\nqzmu1MzMCqAolZqTmplZARQlqbn50czMcsOVmplZzhXpyddOamZmRVCMnObmRzMzyw9XamZmeVeg\n69Sc1MzMCqAoSc3Nj2Zmlhuu1MzMCqAolZqTmplZERQjp7n50czM8sOVmplZAbj50czMckEqzh1F\n3PxoZma54UrNzKwAilKpOamZmRVAUZKamx/NzCw3XKmZmRVBMQo1JzUzsyJw86OZmVmNcaVmZpZ3\nfvSMmZnlhYCC5DQ3P5qZWX64UjMzy73i3CbLSc3MrAAKktPc/GhmZvnhSs3MrACK0vzoSs3MzHLD\nlZqZWd6pOH1qTmpmZjknoFOnYmQ1Nz+amVluuFIzMysANz+amVluePSjmZlZjXGlZmaWdx79aGZm\neZHdpb8YWc3Nj9aiN16fxlH7DWefr23BiB235Jo/XQLAvbfdxIgdt2TzNfvw3NMTP9l+zjuzOWq/\n4Wy7/qr84kcnLfSY3zt8JCN2Gtoh8Vt1+fDDD9l2qy3ZcrON2WyjDfi/H58FwFNPPsmXt/kiWw7Z\nhK2Hbs748Y9VOFKrVa7UrEVdunThe2f8lPU22JgP3n+PA3f7Mltu8xXWWncw515yNT8744RPbd+t\nWzeOPvEMprzwHC9Nfv4zx/v7neNYumfPjgrfqky3bt244+776NWrF3V1dWy/3bbsNGxn/u/HZ/GD\nM3/ETsN25s47bufM00/lrnv/Uelwc6Q4d+l3pWYtWuFzK7PeBhsD0LNXbwYMWpe33nidgYPWZcBa\na39m+x5L92Tjzb9It27dP7Nu3gfv8+crfsdhx53c7nFbdZJEr169AKirq6Ourg6U/cF97913AXh3\n7lxWWWXVSoaZS1L5pmrmSs1a7fVprzD5uafZYOMhi7X/JReM4oDDj6N7jx5ljsxqSX19PVttOYSX\nX5rCUd8+hi222JJzz/8Vuw8fxumnnUxDQwP/+OeDlQ7TapQrNWuVeR+8zylHH8j3f/hzevVeps37\nT37uaaa9MpWv7LRbO0RntaRz5848OuEJXpz6GhMmjOfZZ57hsksv4dzzLuDFl1/l3PMu4OijDq90\nmLmjVBGXY6pm7ZbUJD20GPv8V9KNJfN7Sxpd1sAWHcPZkhY+wqGgFtTVccrRBzJsjxF8ddjui3WM\nSRMf4/lJT7DbNhty+D7DeHXqFI4cuWuZI7Va0qdPH7705e245+47+fNVV7LH1/cCYK+992GCB4qU\nVxmbHqs8p7VfUouIrRZz180kDV6cHSW5ObXMIoKfnHocAwetywGHH7fYx9n7gMO589HJ3PrvSVx+\n/Z2sPnAQl157WxkjtVowc+ZM5syZA8D8+fP5+333ss6667HKKqvywL/+CcD9//g7aw36bH+tWWu0\nWxKQ9H5E9JK0CnAdsEw639ER8UALu/4SOAPYv8nxlgP+CKwJzAOOjIinJZ0NrJWWvyrpLmBPoCew\nNnA+sBRwIPARsEtEzJZ0BHBkWjcFODAi5pXlzefIUxMe4fabrmXQuuvzzV22AeCYk39E3ccfcd7Z\np/DO7FmccOgI1hm8IRddeRMAu22zIR+8/y51dXX8857buOjKm1hz7fUq+TasSrwxYwZHHPYtGurr\naWhoYK+992GXXYfTp08fTjrxBOoXLKBb9+5cdMkfKh1qrhTpOrWOqGy+CdwVEaMkdQaWXsT2Y4Fj\nJA1qsvzHwBMRsaekrwJXAhundYOBbSJivqRvARsAmwDdyRLWqRGxiaRfAQcBvwb+GhGXAUj6KXAY\n8NuWApN0JFkiZOVVV1v0O8+BjTf/IhOmzl3ouub6x27996QWj7lq/zUYe9cjSxyb1Z4Nv/AFHhk/\n8TPLt9p6Gx56dEIFIiqOguS0DhkoMh44JFVUG0bEe4vYvh44Dzi9yfJtgKsAIuLvwPKSGkcsjIuI\n+SXb/iMi3ouImcBc4Na0fBIwIL3eQNIDkiaRVYXrL+qNRMSlETEkIob0XX75RW1uZmYdrN2TWkT8\nC/gSMB0YLemgVux2VdqnteXQB03mPyp53VAy38D/qtPRwHERsSFZFfjZC6vMzHLCox/LRNIawJup\nqe9yYNNF7RMRdcCvgO+VLH6A1M8maTtgVkS8uwSh9QZmSOpKk/47M7O8Kcrox47oU9sOOFlSHfA+\nWZ9Wa1wBnFkyfzbwR0lPkw0UOXgJ4/oh8CgwM/3svYTHMzOzCmu3pBYRvdLPMcCYVu4zoOT1R8Cq\nJfOzyUY1Nt3n7Cbzo8maFhd2zE/WRcQlwCWLOp6ZWc2TRz+amVlOZEP6Kx1Fx6hIUpP0KNCtyeID\nI6LlseBmZmYtqEhSi4gtK3FeM7Niqv5Ri+Xi5kczswIoSE7zXfrNzCw/XKmZmRWAmx/NzCwfauCi\n6XJx86OZmeWGKzUzs5zzo2fMzCxXipLU3PxoZma54UrNzKwAClKoOamZmRWBmx/NzMxqjCs1M7O8\n83VqZmaWF0o3NC7X1KpzSn0k3SDpP5Kel/RFSctJukfSi+ln35LtT5c0RdJkSTuVLN9M0qS07jda\nRABOamZm1h4uBO6MiPWAjYDngdOA+yJibeC+NI+kwcBIYH1gGHCxpM7pOJcARwBrp2lYSyd1UjMz\nKwCpfNOiz6VlgS8BVwBExMcRMQfYAxiTNhsD7Jle7wFcGxEfRcRUYAqwhaRVgGUi4pGICODKkn0W\nyn1qZmYF0KljO9UGAjOBP0naCHgcOB5YKSJmpG3eAFZKr/sBj5TsPy0tq0uvmy5vlis1MzNrqxUk\nTSiZjmyyvguwKXBJRGwCfEBqamyUKq8od2Cu1MzMCqDMhdqsiBjSwvppwLSIeDTN30CW1N6UtEpE\nzEhNi2+l9dOB1Ur275+WTU+vmy5vlis1M7Ocy/rCOm70Y0S8Abwmad20aHvgOWAccHBadjBwS3o9\nDhgpqZukgWQDQh5LTZXvShqaRj0eVLLPQrlSMzOz9vAd4M+SlgJeBg4hK6TGSjoMeAUYARARz0oa\nS5b4FgDHRkR9Os4xwGigB3BHmprlpGZmVgCdOvji64h4ElhYE+X2zWw/Chi1kOUTgA1ae14nNTOz\nAvC9H83MzGqMKzUzswIoSKHmpGZmlnciu/9jEbj50czMcsOVmplZAXT06MdKcVIzM8u7Njwypta5\n+dHMzHLDlZqZWQEUpFBzUjMzyzvR4Y+eqRg3P5qZWW64UjMzK4CCFGpOamZmReDRj2ZmZjXGlZqZ\nWc5lDwmtdBQdw0nNzKwAPPrRzMysxjRbqUlapqUdI+Ld8odjZmbtoRh1WsvNj88Cwac/i8b5AFZv\nx7jMzKyMijL6sdmkFhGrdWQgZmZmS6pVfWqSRkr6QXrdX9Jm7RuWmZmVS3abrPJN1WyRSU3SRcBX\ngAPTonnA79szKDMzK6P06JlyTdWsNUP6t4qITSU9ARARsyUt1c5xmZmZtVlrklqdpE5kg0OQtDzQ\n0K5RmZlZWVV5gVU2rUlqvwNuBFaU9GNgBPDjdo3KzMzKqtqbDctlkUktIq6U9DiwQ1q0T0Q8075h\nmZmZtV1rb5PVGagja4L0XUjMzGpI4+jHImjN6MczgGuAVYH+wF8knd7egZmZWfl49OP/HARsEhHz\nACSNAp4Aft6egZmZmbVVa5LajCbbdUnLzMysRlR3fVU+Ld3Q+FdkfWizgWcl3ZXmdwTGd0x4Zma2\npKTiPHqmpUqtcYTjs8BtJcsfab9wzMzMFl9LNzS+oiMDMTOz9lOQQm3RfWqS1gJGAYOB7o3LI2Kd\ndozLzMyszVpzzdlo4E9k/Yw7A2OB69oxJjMzK7OiDOlvTVJbOiLuAoiIlyLiTLLkZmZmNUIq31TN\nWjOk/6N0Q+OXJH0bmA70bt+wzMzM2q41Se17QE/gu2R9a8sCh7ZnUGZmVj5CHtLfKCIeTS/f438P\nCjUzs1pRA82G5dLSxdc3kZ6htjARsVe7RGRmZraYWqrULuqwKGpQj66dWb//MpUOw2pM382Pq3QI\nVmM+mvxqWY5T7aMWy6Wli6/v68hAzMys/RTlmWFFeZ9mZlYArX1IqJmZ1Sjh5sfPkNQtIj5qz2DM\nzKx9+MnXiaQtJE0CXkzzG0n6bbtHZmZm1kat6VP7DTAceBsgIp4CvtKeQZmZWXl1Uvmmataa5sdO\nEfFKk/bY+naKx8zMyiy7Z2OVZ6MyaU1Se03SFkBI6gx8B3ihfcMyMzNru9YktaPJmiBXB94E7k3L\nzMysRlR7s2G5tObej28BIzsgFjMzaycFaX1s1ZOvL2Mh94CMiCPbJSIzM7PF1Jrmx3tLXncHvg68\n1j7hmJlZuQn86JlGEXFd6bykq4B/t1tEZmZWdkW5J+LivM+BwErlDsTMzGxJtaZP7R3+16fWCZgN\nnNaeQZmZWXkVpPWx5aSm7Gq9jYDpaVFDRDT74FAzM6s+kgrTp9Zi82NKYLdHRH2anNDMzKxqtaZP\n7UlJm7R7JGZm1m6yW2WVZ6pmzTY/SuoSEQuATYDxkl4CPiAbHRoRsWkHxWhmZkvIdxSBx4BNgd07\nKBYzM7Ml0lJSE0BEvNRBsZiZWTvwxdeZFSWd2NzKiLigHeIxM7N2UJCc1mJS6wz0IlVsZmZm1a6l\npDYjIn7SYZGYmVn7qIEnVpfLIvvUzMys9qkgf9Jbuk5t+w6LwszMrAyardQiYnZHBmJmZu0jG/1Y\n6Sg6Rmuep2ZmZjWuKEmtKI/YMTOzAnClZmZWACrIhWqu1MzMcq6xT61cU6vOKXWW9ISkv6X55STd\nI+nF9LNvybanS5oiabKknUqWbyZpUlr3G7UiMzupmZlZezgeeL5k/jTgvohYG7gvzSNpMDASWB8Y\nBlwsqXPa5xLgCGDtNA1b1Emd1MzM8q6Mj51pTSumpP7ArsDlJYv3AMak12OAPUuWXxsRH0XEVGAK\nsIWkVYBlIuKR9CzPK0v2aZb71MzMCqDMNzReQdKEkvlLI+LSkvlfA6cAvUuWrRQRM9LrN4CV0ut+\nwCMl201Ly+rS66bLW+SkZmZmbTUrIoYsbIWk4cBbEfG4pO0Wtk1EhKRoj8Cc1MzMcq6DL77eGthd\n0i5Ad2AZSVcDb0paJSJmpKbFt9L204HVSvbvn5ZNT6+bLm+R+9TMzAqgo/rUIuL0iOgfEQPIBoD8\nPSIOAMYBB6fNDgZuSa/HASMldZM0kGxAyGOpqfJdSUPTqMeDSvZplis1MzPrCOcAYyUdBrwCjACI\niGcljQWeAxYAx0ZEfdrnGGA00AO4I00tclIzM8s90akCd+mPiPuB+9Prt2nmRvkRMQoYtZDlE4AN\n2nJOJzUzs5wTxXnytfvUzMwsN1ypmZnlnZ98bWZmeVLmi6+rlpsfzcwsN1ypmZnlXJEGijipmZkV\ngJsfzczMaowrNTOzAihIoeakZmaWd6I4zXJFeZ9mZlYArtTMzPJOoIK0PzqpmZkVQDFSmpsfzcws\nR1ypmZnlXPbk62LUak5qZmYFUIyU5uZHMzPLEVdqZmYFUJDWRyc1M7P8U2GG9Lv50czMcsOVmplZ\nzhXpNllOamZmBeDmRzMzsxrjSs3MrACKUac5qVkZ3X3XnZx04vHU19fzrUMP5+RTTqt0SFZBx+63\nHYfstRWS+NNfH+Siv9wPwNEjv8xRI7alviG484FnOOPCW+japTMXnbkfmw5enYZo4KRzb+SBx18E\nYMSwzTj50J2ICGbMnMuhZ47h7TkfVO6N1SLf0Nisberr6znhu8dy2x330K9/f7YZujnDh+/O5wcP\nrnRoVgGD11qFQ/baim0PPI+P6+oZ97tjuP2BZ+i/Ul+Gb7chW+x7Dh/XLWDFvr0AOHSvrQHYfMTP\nWLFvL26+6Bi2OeA8OnUS5528N5t+46e8PecDRh2/B9/e98uM+sPtlXx7VsXcp2ZlMf6xx1hrrUEM\nXHNNllpqKfbZdyR/u/WWSodlFbLewJUZ/8x/mf9hHfX1DTzw+BT2/OrGHLnPtpz/p3v4uG4BADPf\neT/bfs2VuX/85E+WzX1vPpsNXh0pu2i4Z4+lAOjdqwczZs6tzJuqYY2jH8s1VbNqj89qxOuvT6d/\n/9U+me/Xrz/Tp0+vYERWSc++9DpbbzKI5ZbtSY/uXRm2zfr0X7kvg9b4HFtvshb/uvIk7r78eDYb\nvDoAk16YzvAvb0jnzp1YY9Xl2WTwavRfuS8LFjRw/M+uY/zYH/Dy3aP4/JorM/rmhyr87mqTpLJN\n1czNj2ZWdpOnvskvR9/DrRcfy7wPP+apydOor2+gS+dOLLdsT7500PkMWX8Nrj73UD4//GzG3PIw\n6w1ciQf/fAqvzpjNI09Nzbbv0okj9t6Wofv9gqnTZvGrU/fh5EN35BeX31Xpt2hVqsMqNUmL9fVK\n0saSQtKwkmV9JB1TMj9A0jeXILb7JQ1Z3P0NVl21H9OmvfbJ/PTp0+jXr18FI7JKG3Pzw2y9/7l8\n7bBfM+fdebz4yltMf3MON9/3JAATnn2FhoZghb69qK9v4JRf/pWhI89hxPcupU/vHrz46ltstE5/\nAKZOmwXADfdMZOhGa1bsPdUylXGqZh2W1CJiq8XcdT/g3+lnoz7AMSXzA4DFTmq25IZsvjlTprzI\nf6dO5eOPP+b6665l1+G7Vzosq6DGQSCrrdyXPb66EdfdMYFb73+aL2++DgCDVv8cS3Xtwqx33qdH\n964s3T3rN/vqluuxoL6B/7z8Bq/PnMt6a67MCulY2w9dj8lT36jMG7Ka0GHNj5Lej4heklYBrgOW\nSec/OiIeaGYfAfsAXwMekNQ9Ij4EzgHWkvQkcA+wLfD5ND8GuAm4CuiZDnVcRDyUjnkqcADQANwR\nEaeVnK8T8EdgWkScuZB4jgSOBFht9dWX6PPImy5duvCrCy9it113or6+noO/dSiD11+/0mFZBV1z\n/uEs16fKPiz8AAASfUlEQVQndQvqOeGcscx9fz5jbn6YP5y9PxOu/wEf19Vz+I+uAmDFvr259eJj\naWgIXp85h8POHAPAjJlz+dmld3DP5SdQt6CeV2fM5sizrq7k26pZVd4VVjaKiI450f+S2veB7hEx\nSlJnYOmIeK+ZfbYGfhIR20v6C3BjRNwoaQDwt4jYIG23HXBSRAxP80sDDRHxoaS1gWsiYoiknYEf\nAjtExDxJy0XEbEn3A6cBxwPPRMSoRb2fzTYbEg8+OmGJPhMrnr6bH1fpEKzGfDR5LA3z3lqilLT2\n+hvFBdfeXa6Q2P0LKz8eEVXZZVOJ0Y/jgUMknQ1s2FxCS/YDrk2vr+XTTZAt6QpcJmkScD3QeLHU\nDsCfImIeQETMLtnnD7QyoZmZWXXq8KQWEf8CvgRMB0ZLOmhh26Uq7hvAjyT9F/gtMExS71ac5nvA\nm8BGwBBgqVbs8xDwFUndW7GtmVlNabzmrxxTNevwpCZpDeDNiLgMuBzYtJlNtweejojVImJARKwB\n3Ah8HXgPKE1uTeeXBWZERANwINA5Lb+HrEpcOsWyXMk+VwC3A2Ml+VIHM8sRlfW/alaJ5sftgKck\nPQHsC1zYzHb7kQ34KHUjsF9EvA08KOkZSecBTwP1kp6S9D3gYuBgSU8B6wEfAETEncA4YEIaVHJS\n6cEj4gLgCeCqNGjEzMxqSIdVJBHRK/0cQzZCcVHbH7KQZePIkhIR0XQI/1ebzH+h5PWpJcc4h2z0\nZOlxtyt5fdaiYjMzqzXV3mxYLm5mMzPLuezej8XIalWR1CQ9CnRrsvjAiJhUiXjMzKw2VUVSi4gt\nKx2DmVlu1cCoxXKpiqRmZmbtqyhJzSP8zMwsN1ypmZkVQLVfX1YuTmpmZjknoFMxcpqbH83MLD9c\nqZmZFYCbH83MLDc8+tHMzKzGuFIzMysANz+amVkuePSjmZlZDXKlZmaWe9X/cM9ycVIzM8u7At3Q\n2M2PZmaWG67UzMwKoCCFmpOamVneZaMfi5HW3PxoZma54UrNzKwAilGnOamZmRVDQbKamx/NzCw3\nXKmZmRWAL742M7PcKMjgRzc/mplZfrhSMzMrgIIUak5qZmaFUJCs5uZHMzPLDVdqZmY5Jzz60czM\n8sKPnjEzM6s9TmpmZgWgMk6LPJe0mqR/SHpO0rOSjk/Ll5N0j6QX08++JfucLmmKpMmSdipZvpmk\nSWndb6SWa04nNTOzIujIrAYLgO9HxGBgKHCspMHAacB9EbE2cF+aJ60bCawPDAMultQ5HesS4Ahg\n7TQNa+nETmpmZlZWETEjIiam1+8BzwP9gD2AMWmzMcCe6fUewLUR8VFETAWmAFtIWgVYJiIeiYgA\nrizZZ6E8UMTMLPdUsdGPkgYAmwCPAitFxIy06g1gpfS6H/BIyW7T0rK69Lrp8mY5qZmZFUCZRz+u\nIGlCyfylEXHpZ8+pXsCNwAkR8W5pd1hEhKQoa1Q4qZmZWdvNioghLW0gqStZQvtzRPw1LX5T0ioR\nMSM1Lb6Vlk8HVivZvX9aNj29brq8We5TMzPLuXKOEWnl6EcBVwDPR8QFJavGAQen1wcDt5QsHymp\nm6SBZANCHktNle9KGpqOeVDJPgvlSs3MrAg6tktta+BAYJKkJ9OyHwDnAGMlHQa8AowAiIhnJY0F\nniMbOXlsRNSn/Y4BRgM9gDvS1CwnNTMzK6uI+DfNp9Htm9lnFDBqIcsnABu09txOamZmBeB7P5qZ\nWW743o9mZmY1xpWamVkBFKRQc1IzM8u91t+zsea5+dHMzHLDlZqZWQF49KOZmeWC8OhHMzOzmuNK\nzcysAApSqDmpmZkVQkGympsfzcwsN1ypmZkVgEc/mplZbnj0o5mZWY1xpWZmVgAFKdSc1MzMCqEg\nWc3Nj2Zmlhuu1MzMci67SX8xSjUnNTOzvJNHP5qZmdUcV2pmZgVQkELNSc3MrBAKktWc1BbTxImP\nz+rRVa9UOo4qtAIwq9JBWM3x703z1qh0ALXESW0xRcSKlY6hGkmaEBFDKh2H1Rb/3rQ3efSjmZnl\nh0c/mpmZ1RhXalZul1Y6AKtJ/r1pR6Iw40Sc1Ky8IsJ/nKzN/HvTAQqS1dz8aGZmueFKzcysAIoy\n+tGVmpmZ5YYrNas4ScsBK0TEC5WOxWqPJEVEVDqOauch/WYdQFJ34LvAoZI+X+l4rHZIWg3ACa11\nVMapmjmpWUVFxIfAvWl2H0mDKxmPVS9JvSQtlV5/HjhXUu8Kh2VVxknNKkbKGkQi4t/AOGAZYG8n\nNmtKUk/gz8A+adG8NL0vqWvaptqLiMpJz1Mr11TNnNSsIhr7QSQNlNQlIh4C/gQsS5bY3BRpn4iI\nD4DrgEMk7QsMAOZHpi5t42bIFhWjAdIDRawiUkLbFfgh8ICk94Ffk91Z4jDgAEl/jojnKhmnVZ6k\nzhFRHxF/kTQTOBV4HBgo6UJgGvAR0CUiLqhkrFZ5rtSsIiQNBX4G7Ev25WpP4FxgJjAG6Al8XLEA\nrSqkir5e0tcknRsR9wAXAtuT/X68mn72Ah6tYKhVTRSn+dGVmnUoSZ2AIHt+1kHAesCXgNOAI4Hz\nyb6Jn5GanKzAUkW/PXAxcFRadqukBcCJwAsRcWslY6wVVZ6LysaVmnWIkk78Xqkf5G8R8RRZhXZ4\nRNwFvEX2RWslJzRTpgswDPhhRPy9cfRjRNwB/B44VVK/SsZp1cVJzTpESR/afZLOlrRXWvU54EhJ\nWwJbAOdHxDMVC9SqRvryswD4EBgqqXtEfAwgaXPgdmD3iJheyThrRVGaH53UrENIWgXYn6x5cTaw\nU0pyhwKrAT8Cfh4RT1cuSqu0xope0uqS+qfFdwBdgS+ndRsBvwLWiYjZFQm0BqmM/1Uz96lZu5M0\nBNgImB4R10laEdgJ+DrQNSKGS1o6Iub5lkfFVlLR/xx4SNJyETEiXeJxoKRTyS77+Glqvjb7FCc1\na1eStiMbzXgX2TD9ayJioqQ7gKWAPSQ9FhGvg681KqqS6xaHko2CHU5Wmf1R0r0RsYOk0WRfjuZG\nxEv+AtRG1V1glY2TmrUbSQOBHwAHRsS/JE0Brpa0f0Q8IekW4M7GhGbFk+79WZeG7a8EvA2MANYm\nG+24LHC/pIciYitgYuO+TmhtU5Cc5j41K6+SPpHNyb5pL0s2wpGIOBe4AhgnabOIeNsJrbjS5R1b\nASdIGk7Wr/oe8BywK/DHiHiPrNJfPf1OmbXISc3KKjUhfYmsCWkS2QXWS0s6Lq3/JfA7sotlzZ4G\ndgSuAm6IiDfIiooZwFqSjiBrivxaRIyvXJi1rZwjHz360QpF0rrA0cDoiHgcuB+4D1hP0vcBIuKc\niPinb0BbTJJ6SuofEQ3AGmnxP4Cd07D9BrInN8wjS2i/j4jnKxRubhRl9KOTmpXbhsBKwA6SVoyI\nucCdwEPAupIa/4i5T6S4BgC/lXQGcBLwfeA7ZE9paLx348tkie4bEfFXfwGy1nJSsyVS0ofWX9Ky\nEXED2U2K3yW72/7yqV/kVuBHEfFKBcO1KhARzwJTyAYRPZoutp9JdiusbpLuI6vw69LF1/4CVA7F\nuEm/Rz/a4pPUKSIaJO1M1oc2WdLnyAaG/A3Ymezaoqsi4m2yQQBWQJL6AB9HxLy06Bngl8BBkiZF\nxH3A06l6+xrwekQ8UqFwc6nKc1HZOKlZm0nqERHzU0IbBPwfcFREPCTpN8DNZBdXd00/e5IN1bYC\nkrQc8AJwr6QHIuJ3ETEmrXsNuEDSwcAcYK/Gx8f4OjRbHE5q1iaSlgXOkXRTRNxN9ofoP2R/tIiI\n70q6BjgtIs6SND4iZlQwZKu8d4C7yUY07i9pC+DfwPURcZmkj4EbgQXACY07OaGVV1F6Jd2nZm21\nDFl/yDfTI0HeBZYHdijZ5nbSs9Cc0Cwlp4lkA4i+BIxOP/8p6StkA0K2JBsUckel4sy3co59rO7s\n6ErNWkVS74h4LyJek3QlMJLsZsQzyTr8R0taD5iblp9SuWit2kTE+ZJuJ/vy8wywMVmFPxIYBOzr\npzNYOTip2SJJGgDcIOlxYCzwIvAn4COy4di/APYhGxiyKvC9iLjXfSIGIKlzRNSTVWhfJ7vD/hUp\n0X2O7KbWsyoZY941Pvm6CJzUrDW6A6sAewD/JbsjyO+BvmTXn/0QGBURF5bu5IRmACmhATwKnA08\nHBHnp2Uz/Xti5eQ+NWtRGrb/H7Jmo7nAq8C+wOtk93bcO82fK6lPup+f2aekqv0V4ESgV+PTqp3Q\nrNxcqVmL0rD9ThHxvKQDgGuBn0XEFZJuILub+h7AkxExp6LBWkWVPD6mU7rV1SdKktc0oOGze1t7\nc/OjWVKS2MZLGglck+7R9ztgMtmF176uqMBKEtr2ZJXYXRHxYdPtIuIZSadGxPQKhFlo1T5qsVzc\nVGStUprYyJobfyjp2CbbOKEVUBoIEpKGAZcA7ywsoSnTKSJekbS0pOU7PlrLOyc1+5SSezl+5nej\nJLE9DuwGPNvR8Vn1kDQoXepRL6kv2YChb6cHwm4r6eB0oXWjxtuq9SG7Nm25igReRAV69IybH+0T\nrWlCalKxucmx2FYCPifpkYh4R9I/gMPSM9A6AXVkfa6PSeoSEQvSHWmuB06OiBcrF3qx1MB9iMvG\nlZoBrW9Catw87dODbFi/FVBEPEj2INiXJS1Ddh3aY8BvI2Jfsmsa15e0VEpofYGbgJ9ExL8qFbfl\nm5NawbW1CanxQtrUhHQ/2S2yrKDSY4WOJ7tecVZEXJhubL0t2Y2uL4+Ij9Pm+wE/jYgHKhRusfnR\nM1YQbkKyJRIRt0iqAx6XtBnwIdn1i2dGxG2NTdQRcXFlIy22oox+dFIruIh4UFJvsiakL5A1Ie0K\njE/fuHcHDklNSB+nau5G4Cx/47ZGEXG7pAbgeWBd4NSI+LCkn9Z9r9Yh3PxobkKysoiIO4HDgU0a\n+2MbE5kTWuV59KMVipuQrBwi4jbwqNhqVOW5qGyc1OwTbkKycvHviVWKmx/tU9yEZJZTHTj6UdIw\nSZMlTZF0WrnfSktcqdlnuAnJLH86avSjpM5kj6f6GtkNrMdLGhcRz3XE+V2pWbOc0MxsMWwBTImI\nl9MAs2vJnuTRIVypmZnlXAc/+bof8FrJ/DRgy446uZOamVnOTZz4+F09umqFMh6yu6QJJfOXRsSl\nZTz+YnNSMzPLuYgY1oGnmw6sVjLfPy3rEO5Ts9ySVC/pSUnPSLpe0tJLcKztJP0tvd69pRFdkvpI\nOmYxznG2pJNau7zJNqMl7d2Gcw2Q9ExbYzRrhfHA2pIGSloKGAmM66iTO6lZns2PiI0jYgPgY+Db\npSsbH1rZ1oNGxLiIOKeFTfoAbU5qZnkQEQuA44C7yK55HRsRHfbsRSc1K4oHgEGpQpks6UrgGWA1\nSTtKeljSxFTR9YJPrrX5j6SJwF6NB5L0LUkXpdcrSbpJ0lNp2go4B1grVYnnpe1OljRe0tOSflxy\nrDMkvSDp32QXvLdI0hHpOE9JurFJ9bmDpAnpeMPT9p0lnVdy7qOW9IM0W5SIuD0i1omItSJiVEee\n20nNck9SF2Bnsmd/QfbUgYsjYn3gA+BMYIeI2BSYAJwoqTtwGdkTvjcDVm7m8L8B/hkRGwGbkj0N\n/DTgpVQlnixpx3TOLYCNgc0kfSndjmxkWrYLsHkr3s5fI2LzdL7ngcNK1g1I59gV+H16D4cBcyNi\n83T8IyQNbMV5zGqSB4pYnvWQ9GR6/QBwBbAq8EpEPJKWDwUGAw8qG/O8FPAwsB4wtfHROpKuBo5c\nyDm+ChwEEBH1wNz0JINSO6bpiTTfiyzJ9QZuioh56Ryt6XfYQNJPyZo4e5E18TQaGxENwIuSXk7v\nYUfgCyX9bcumc7/QinOZ1RwnNcuz+RGxcemClLg+KF0E3BMR+zXZ7lP7LSEBP4+IPzQ5xwmLcazR\nwJ4R8ZSkbwHblaxrerF8pHN/JyJKkx+SBizGuc2qnpsfregeAbaWNAhAUk9J6wD/AQZIWittt18z\n+98HHJ327azsAarvkVVhje4CDi3pq+sn6XPAv4A9JfVQ9ky73VoRb29ghqSuwP5N1u0jqVOKeU1g\ncjr30Wl7JK0jqWcrzmNWk1ypWaFFxMxU8VwjqVtafGZEvCDpSOA2SfPImi97L+QQxwOXSjoMqAeO\njoiHJT2YhszfkfrVPg88nCrF94EDImKipOuAp4C3yIZCL8oPgUeBmelnaUyvAo8BywDfTk9YuJys\nr22ispPPBPZs3adjVnvk2/uZmVleuPnRzMxyw0nNzMxyw0nNzMxyw0nNzMxyw0nNzMxyw0nNzMxy\nw0nNzMxyw0nNzMxy4/8BZ4zRZvIQlEcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbd784627f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = Train.actual_value_, pred_value = Train.pred_value_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-16T20:42:20.473337Z",
     "start_time": "2017-06-16T20:42:20.433139Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#4.5 GB\n",
    "pd.Series(Train.pred_value).to_csv('GRU_prediction_values_.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
